{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "86a4bd36",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "下面展示基于隐马尔科夫模型的序列标注监督学习的代码。这里以命名实体识别任务为例，所使用的数据是Books数据集。为简单起见，标签序列采用BIO格式。首先构建数据集和标签集合："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "41673bbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'B-NAME': 1, 'I-ORG': 2, 'B-PRO': 3, 'B-EDU': 4, 'I-NAME': 5, 'B-LOC': 6, 'B-TITLE': 7, 'B-RACE': 8, 'I-TITLE': 9, 'I-RACE': 10, 'I-PRO': 11, 'B-CONT': 12, 'I-EDU': 13, 'I-LOC': 14, 'I-CONT': 15, 'B-ORG': 16, 'O': 0}\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "from collections import defaultdict\n",
    "\n",
    "\n",
    "class NERDataset:\n",
    "    def __init__(self):\n",
    "        train_file, test_file, label_file = 'ner_train.jsonl',\\\n",
    "            'ner_test.jsonl', 'ner_labels.json'\n",
    "        \n",
    "        def read_file(file_name):\n",
    "            with open(file_name, 'r', encoding='utf-8') as fin:\n",
    "                json_list = list(fin)\n",
    "            data_split = []\n",
    "            for json_str in json_list:\n",
    "                raw = json.loads(json_str)\n",
    "                d = {'tokens': raw['tokens'], 'tags': raw['tags']}\n",
    "                data_split.append(d)\n",
    "            return data_split\n",
    "        \n",
    "        # 读取JSON文件，转化为Python对象\n",
    "        self.train_data, self.test_data = read_file(train_file),\\\n",
    "            read_file(test_file)\n",
    "        self.label2id = json.loads(open(label_file, 'r').read())\n",
    "        self.id2label = {}\n",
    "        for k, v in self.label2id.items():\n",
    "            self.id2label[v] = k\n",
    "        print(self.label2id)\n",
    "\n",
    "    # 建立词表，过滤低频词\n",
    "    def build_vocab(self, min_freq=3):\n",
    "        # 统计词频\n",
    "        frequency = defaultdict(int)\n",
    "        for data in self.train_data:\n",
    "            tokens = data['tokens']\n",
    "            for token in tokens:\n",
    "                frequency[token] += 1\n",
    "                \n",
    "        print(f'unique tokens = {len(frequency)}, '+\\\n",
    "              f'total counts = {sum(frequency.values())}, '+\\\n",
    "              f'max freq = {max(frequency.values())}, '+\\\n",
    "              f'min freq = {min(frequency.values())}')\n",
    "        \n",
    "        # 由于词与标签一一对应，不能随便删除字符，\n",
    "        # 因此加入<unk>用于替代未登录词，加入<pad>用于批量训练\n",
    "        self.token2id = {'<pad>': 0, '<unk>': 1}\n",
    "        self.id2token = {0: '<pad>', 1: '<unk>'}\n",
    "        total_count = 0\n",
    "        # 将高频词加入词表\n",
    "        for token, freq in sorted(frequency.items(),\\\n",
    "                key=lambda x: -x[1]):\n",
    "            if freq > min_freq:\n",
    "                self.token2id[token] = len(self.token2id)\n",
    "                self.id2token[len(self.id2token)] = token\n",
    "                total_count += freq\n",
    "            else:\n",
    "                break\n",
    "        print(f'min_freq = {min_freq}, '\n",
    "              f'remaining tokens = {len(self.token2id)}, '\n",
    "              f'in-vocab rate = {total_count / sum(frequency.values())}')\n",
    "\n",
    "    # 将文本输入转化为词表中对应的索引\n",
    "    def convert_tokens_to_ids(self):\n",
    "        for data_split in [self.train_data, self.test_data]:\n",
    "            for data in data_split:\n",
    "                data['token_ids'] = []\n",
    "                for token in data['tokens']:\n",
    "                    data['token_ids'].append(self.token2id.get(token, 1)) \n",
    "        \n",
    "dataset = NERDataset()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25650a7c",
   "metadata": {},
   "source": [
    "接下来建立词表，将词元转换为索引，并将数据转化为适合训练的格式。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a3eb49e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique tokens = 2454, total counts = 182068, max freq = 5630, min freq = 1\n",
      "min_freq = 0, remaining tokens = 2456, in-vocab rate = 1.0\n",
      "[595, 510, 353, 263, 424, 225, 764, 637, 353, 65, 80, 47, 41, 74, 53, 41, 141, 23, 74, 7, 259, 27, 135, 47, 31, 74, 67, 25, 1605, 815, 1157, 225, 595, 1158, 738, 688, 353, 1025, 65, 234, 27, 135, 316, 41, 31, 7, 80, 41, 42, 31, 47, 27, 528, 41, 141, 27, 135, 67, 1005, 1606, 363, 19, 677, 294, 37, 678, 1756, 604, 873, 1006, 1005, 353, 33, 30, 9, 113, 5, 45, 11, 82, 60, 21, 26, 5, 186, 153, 100, 565, 873, 353, 424, 1993, 387, 346, 250, 13, 5, 102, 214, 25, 11, 82, 60, 21, 26, 5, 186, 2, 7, 459, 97, 89, 147, 140, 139, 69, 46, 21, 12, 2, 7, 964, 98, 240, 677, 59, 9, 103, 11, 82, 60, 335, 200, 76, 26, 6, 110, 20, 9, 15, 4, 122, 539, 241, 99, 621, 40, 200, 57, 82, 156, 25, 11, 82, 381, 371, 91, 202, 6, 52, 2, 7, 670, 100, 84, 204, 53, 120, 27, 42, 3, 56, 1159, 3, 329, 31, 265, 31, 3, 56, 394, 394, 3, 84, 357, 84, 7, 25, 397, 7, 41, 7, 74, 7, 135, 7, 31, 7, 87, 7, 259, 7, 31, 7, 74, 7, 41, 7, 131, 7, 28, 289, 385, 4]\n",
      "['阿', '里', '斯', '提', '德', '·', '波', '拉', '斯', '（', 'A', 'r', 'i', 's', 't', 'i', 'd', 'e', 's', ' ', 'B', 'o', 'u', 'r', 'a', 's', '）', '和', '卢', '卡', '雅', '·', '阿', '伊', '纳', '罗', '斯', '托', '（', 'L', 'o', 'u', 'k', 'i', 'a', ' ', 'A', 'i', 'n', 'a', 'r', 'o', 'z', 'i', 'd', 'o', 'u', '）', '夫', '妇', '二', '人', '均', '拥', '有', '希', '腊', '比', '雷', '埃', '夫', '斯', '技', '术', '教', '育', '学', '院', '计', '算', '机', '工', '程', '学', '位', '以', '及', '色', '雷', '斯', '德', '谟', '克', '利', '特', '大', '学', '电', '子', '和', '计', '算', '机', '工', '程', '学', '位', '，', ' ', '都', '从', '事', '过', '软', '件', '开', '发', '工', '作', '，', ' ', '且', '目', '前', '均', '为', '教', '授', '计', '算', '机', '相', '关', '课', '程', '的', '高', '中', '教', '师', '。', '他', '们', '写', '了', '很', '多', '关', '于', '算', '法', '和', '计', '算', '思', '维', '方', '面', '的', '书', '，', ' ', '涉', '及', 'P', 'y', 't', 'h', 'o', 'n', '、', 'C', '#', '、', 'J', 'a', 'v', 'a', '、', 'C', '+', '+', '、', 'P', 'H', 'P', ' ', '和', 'V', ' ', 'i', ' ', 's', ' ', 'u', ' ', 'a', ' ', 'l', ' ', 'B', ' ', 'a', ' ', 's', ' ', 'i', ' ', 'c', ' ', '等', '语', '言', '。']\n"
     ]
    }
   ],
   "source": [
    "# 截取一部分数据便于更快训练\n",
    "dataset.train_data = dataset.train_data[:1000]\n",
    "dataset.test_data = dataset.test_data[:200]\n",
    "\n",
    "dataset.build_vocab(min_freq=0)\n",
    "dataset.convert_tokens_to_ids()\n",
    "print(dataset.train_data[0]['token_ids'])\n",
    "print([dataset.id2token[token_id] for token_id in \\\n",
    "       dataset.train_data[0]['token_ids']])\n",
    "\n",
    "def collect_data(data_split):\n",
    "    X, Y = [], []\n",
    "    for data in data_split:\n",
    "        assert len(data['token_ids']) == len(data['tags'])\n",
    "        X.append(data['token_ids'])\n",
    "        Y.append(data['tags'])\n",
    "    return X, Y\n",
    "\n",
    "train_X, train_Y = collect_data(dataset.train_data)\n",
    "test_X, test_Y = collect_data(dataset.test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7de17192",
   "metadata": {},
   "source": [
    "随后实现隐马尔科夫模型，使用最大似然估计得到模型参数，使用维特比算法进行解码。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3229fd6a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class HMM:\n",
    "    def __init__(self, n_tags, n_tokens):\n",
    "        self.n_tags = n_tags\n",
    "        self.n_tokens = n_tokens\n",
    "    \n",
    "    # 使用最大似然估计计算模型参数\n",
    "    def fit(self, X, Y):\n",
    "        Y0_cnt = np.zeros(self.n_tags)\n",
    "        YY_cnt = np.zeros((self.n_tags, self.n_tags))\n",
    "        YX_cnt = np.zeros((self.n_tags, self.n_tokens))\n",
    "        for x, y in zip(X, Y):\n",
    "            Y0_cnt[y[0]] += 1\n",
    "            last_y = y[0]\n",
    "            for i in range(1, len(y)):\n",
    "                YY_cnt[last_y, y[i]] += 1\n",
    "                last_y = y[i]\n",
    "            for xi, yi in zip(x, y):\n",
    "                YX_cnt[yi, xi] += 1\n",
    "        self.init_prob = Y0_cnt / Y0_cnt.sum()\n",
    "        self.transition_prob = YY_cnt\n",
    "        self.emission_prob = YX_cnt\n",
    "        for i in range(self.n_tags):\n",
    "            # 为了避免训练集过小时除0\n",
    "            yy_sum = YY_cnt[i].sum()\n",
    "            if yy_sum > 0:\n",
    "                self.transition_prob[i] = YY_cnt[i] / yy_sum\n",
    "            yx_sum = YX_cnt[i].sum()\n",
    "            if yx_sum > 0:\n",
    "                self.emission_prob[i] = YX_cnt[i] / yx_sum\n",
    "    \n",
    "    # 已知模型参数的条件下，使用维特比算法解码得到最优标签序列\n",
    "    def viterbi(self, x):\n",
    "        assert hasattr(self, 'init_prob') and hasattr(self,\\\n",
    "            'transition_prob') and hasattr(self, 'emission_prob')\n",
    "        Pi = np.zeros((len(x), self.n_tags))\n",
    "        Y = np.zeros((len(x), self.n_tags), dtype=np.int32)\n",
    "        # 初始化\n",
    "        for i in range(self.n_tags):\n",
    "            Pi[0, i] = self.init_prob[i] * self.emission_prob[i, x[0]]\n",
    "            Y[0, i] = -1\n",
    "        for t in range(1, len(x)):\n",
    "            for i in range(self.n_tags):\n",
    "                tmp = []\n",
    "                for j in range(self.n_tags):\n",
    "                    tmp.append(self.transition_prob[j, i] * Pi[t-1, j])\n",
    "                best_j = np.argmax(tmp)\n",
    "                # 维特比算法递推公式\n",
    "                Pi[t, i] = self.emission_prob[i, x[t]] * tmp[best_j]\n",
    "                Y[t, i] = best_j\n",
    "        y = [np.argmax(Pi[-1])]\n",
    "        for t in range(len(x)-1, 0, -1):\n",
    "            y.append(Y[t, y[-1]])\n",
    "        return np.max(Pi[len(x)-1]), y[::-1]\n",
    "    \n",
    "    def decode(self, X):\n",
    "        Y = []\n",
    "        for x in X:\n",
    "            _, y = self.viterbi(x)\n",
    "            Y.append(y)\n",
    "        return Y\n",
    "\n",
    "hmm = HMM(len(dataset.label2id), len(dataset.token2id))\n",
    "hmm.fit(train_X, train_Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32868ed6",
   "metadata": {},
   "source": [
    "最后验证模型效果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4f4b7e88",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.45762149610217284, recall = 0.3289222699093944, f1 = 0.38274259554692375\n",
      "precision = 0.4189636163175303, recall = 0.23270055113288426, f1 = 0.2992125984251969\n"
     ]
    }
   ],
   "source": [
    "def extract_entity(labels):\n",
    "    entity_list = []\n",
    "    entity_start = -1\n",
    "    entity_length = 0\n",
    "    entity_type = None\n",
    "    for token_index, label in enumerate(labels):\n",
    "        if label.startswith('B'):\n",
    "            if entity_start != -1:\n",
    "                # 遇到了一个新的B，将上一个实体加入列表\n",
    "                entity_list.append((entity_start, entity_length,\\\n",
    "                    entity_type))\n",
    "            # 记录新实体\n",
    "            entity_start = token_index\n",
    "            entity_length = 1\n",
    "            entity_type = label.split('-')[1]\n",
    "        elif label.startswith('I'):\n",
    "            if entity_start != -1:\n",
    "                # 上一个实体未关闭，遇到了一个新的I\n",
    "                if entity_type == label.split('-')[1]:\n",
    "                    # 若上一个实体与当前类型相同，长度+1\n",
    "                    entity_length += 1\n",
    "                else:\n",
    "                    # 若上一个实体与当前类型不同，\n",
    "                    # 将上一个实体加入列表，重置实体\n",
    "                    entity_list.append((entity_start, entity_length,\\\n",
    "                        entity_type))\n",
    "                    entity_start = -1\n",
    "                    entity_length = 0\n",
    "                    entity_type = None\n",
    "        else:\n",
    "            if entity_start != -1:\n",
    "                # 遇到了一个新的O，将上一个实体加入列表\n",
    "                entity_list.append((entity_start, entity_length,\\\n",
    "                    entity_type))\n",
    "            # 重置实体\n",
    "            entity_start = -1\n",
    "            entity_length = 0\n",
    "            entity_type = None\n",
    "    if entity_start != -1:\n",
    "        # 将上一个实体加入列表\n",
    "        entity_list.append((entity_start, entity_length, entity_type))\n",
    "    return entity_list\n",
    "\n",
    "def compute_metric(Y, P):\n",
    "    true_entity_set = set()\n",
    "    pred_entity_set = set()\n",
    "    for sent_no, labels in enumerate(Y):\n",
    "        labels = [dataset.id2label[x] for x in labels]\n",
    "        for ent in extract_entity(labels):\n",
    "            true_entity_set.add((sent_no, ent))\n",
    "    for sent_no, labels in enumerate(P):\n",
    "        labels = [dataset.id2label[x] for x in labels]\n",
    "        for ent in extract_entity(labels):\n",
    "            pred_entity_set.add((sent_no, ent))\n",
    "    if len(true_entity_set) > 0:\n",
    "        recall = len(true_entity_set & pred_entity_set)\\\n",
    "            / len(true_entity_set)\n",
    "    else:\n",
    "        recall = 0\n",
    "    if len(pred_entity_set) > 0:\n",
    "        precision = len(true_entity_set & pred_entity_set)\\\n",
    "            / len(pred_entity_set)\n",
    "    else:\n",
    "        precision = 0\n",
    "    if precision > 0 and recall > 0:\n",
    "        f1 = 2 * precision * recall / (precision + recall)\n",
    "    else:\n",
    "        f1 = 0\n",
    "    return precision, recall, f1\n",
    "\n",
    "train_P = hmm.decode(train_X)\n",
    "p, r, f = compute_metric(train_Y, train_P)\n",
    "print(f'precision = {p}, recall = {r}, f1 = {f}')\n",
    "test_P = hmm.decode(test_X)\n",
    "p, r, f = compute_metric(test_Y, test_P)\n",
    "print(f'precision = {p}, recall = {r}, f1 = {f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3abe8397",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "下面展示基于条件随机场的序列标注模型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "787f8ad1",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "代码修改自GitHub项目kmkurn/pytorch-crf\n",
    "（Copyright (c) 2019, Kemal Kurniawan, MIT License（见附录））\n",
    "\"\"\"\n",
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "class CRFLayer(nn.Module):\n",
    "    def __init__(self, n_tags, n_features):\n",
    "        super().__init__()\n",
    "        self.n_tags = n_tags\n",
    "        self.n_features = n_features\n",
    "        # 定义模型参数\n",
    "        self.transitions = nn.Parameter(torch.empty(\\\n",
    "            n_tags, n_tags))\n",
    "        self.emission_weight = nn.Parameter(torch.empty(\\\n",
    "            n_features, n_tags))\n",
    "        self.start_transitions = nn.Parameter(torch.empty(n_tags))\n",
    "        self.end_transitions = nn.Parameter(torch.empty(n_tags))\n",
    "        self.reset_parameters()\n",
    "        \n",
    "    # 使用（-0.1,0.1）之间的均匀分布初始化参数\n",
    "    def reset_parameters(self):\n",
    "        nn.init.uniform_(self.transitions, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.emission_weight, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.start_transitions, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.end_transitions, -0.1, 0.1)\n",
    "    \n",
    "    # 使用动态规划计算得分\n",
    "    def compute_score(self, emissions, tags, masks):\n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        score = self.start_transitions[tags[0]] + \\\n",
    "            emissions[0, torch.arange(batch_size), tags[0]]\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            score += self.transitions[tags[i-1], tags[i]] * masks[i]\n",
    "            score += emissions[i, torch.arange(batch_size),\\\n",
    "                tags[i]] * masks[i]\n",
    "        \n",
    "        seq_ends = masks.long().sum(dim=0) - 1\n",
    "        last_tags = tags[seq_ends, torch.arange(batch_size)]\n",
    "        score += self.end_transitions[last_tags]\n",
    "        return score\n",
    "    \n",
    "    # 计算配分函数\n",
    "    def computer_normalizer(self, emissions, masks):\n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        # batch_size * n_tags, [起始分数 + y_0为某标签的发射分数 ...]\n",
    "        score = self.start_transitions + emissions[0]\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            # batch_size * n_tags * 1 [y_{i-1}为某tag的总分]\n",
    "            broadcast_score = score.unsqueeze(2)\n",
    "            # batch_size * 1 * n_tags [y_i为某标签的发射分数]\n",
    "            broadcast_emissions = emissions[i].unsqueeze(1)\n",
    "            # batch_size * n_tags * n_tags [任意y_{i-1}到y_i的总分]\n",
    "            next_score = broadcast_score + self.transitions +\\\n",
    "                broadcast_emissions\n",
    "            # batch_size * n_tags [对y_{i-1}求和]\n",
    "            next_score = torch.logsumexp(next_score, dim=1)\n",
    "            # masks为True则更新，否则保留\n",
    "            score = torch.where(masks[i].unsqueeze(1), next_score, score)\n",
    "            \n",
    "        score += self.end_transitions\n",
    "        return torch.logsumexp(score, dim=1)\n",
    "    \n",
    "    def forward(self, features, tags, masks):\n",
    "        \"\"\"\n",
    "        features: seq_len * batch_size * n_features\n",
    "        tags/masks: seq_len * batch_size\n",
    "        \"\"\"\n",
    "        _, batch_size, _ = features.size()\n",
    "        emissions = torch.matmul(features, self.emission_weight)\n",
    "        masks = masks.to(torch.bool)\n",
    "        \n",
    "        score = self.compute_score(emissions, tags, masks)\n",
    "        partition = self.computer_normalizer(emissions, masks)\n",
    "        \n",
    "        likelihood = score - partition\n",
    "        return likelihood.sum() / batch_size\n",
    "    \n",
    "    def decode(self, features, masks):\n",
    "        # 与computer_normalizer类似，sum变为max\n",
    "        emissions = torch.matmul(features, self.emission_weight)\n",
    "        masks = masks.to(torch.bool)\n",
    "        \n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        score = self.start_transitions + emissions[0]\n",
    "        history = []\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            broadcast_score = score.unsqueeze(2)\n",
    "            broadcast_emission = emissions[i].unsqueeze(1)\n",
    "            \n",
    "            next_score = broadcast_score + self.transitions +\\\n",
    "                broadcast_emission\n",
    "            next_score, indices = next_score.max(dim=1)\n",
    "            \n",
    "            score = torch.where(masks[i].unsqueeze(1), next_score,\\\n",
    "                score)\n",
    "            history.append(indices)\n",
    "            \n",
    "        score += self.end_transitions\n",
    "        seq_ends = masks.long().sum(dim=0) - 1\n",
    "        best_tags_list = []\n",
    "        \n",
    "        for idx in range(batch_size):\n",
    "            _, best_last_tag = score[idx].max(dim=0)\n",
    "            best_tags = [best_last_tag.item()]\n",
    "            \n",
    "            for hist in reversed(history[:seq_ends[idx]]):\n",
    "                best_last_tag = hist[idx][best_tags[-1]]\n",
    "                best_tags.append(best_last_tag.item())\n",
    "                \n",
    "            best_tags.reverse()\n",
    "            best_tags_list.append(best_tags)\n",
    "            \n",
    "        return best_tags_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "69f4af96",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CRF(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, n_tags):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "        self.crf = CRFLayer(n_tags, hidden_size)\n",
    "        \n",
    "    def forward(self, input_ids, masks, labels):\n",
    "        \"\"\"\n",
    "        input_ids/masks/labels: batch_size * seq_len\n",
    "        \"\"\"\n",
    "        # 将输入序列转化为词嵌入序列\n",
    "        # batch_size * seq_len * embed_size\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        labels = torch.transpose(labels, 0, 1)\n",
    "        # 隐状态和标签、掩码输入条件随机场计算损失\n",
    "        llh = self.crf(embed, labels, masks)\n",
    "        return -llh\n",
    "    \n",
    "    def decode(self, input_ids, masks):\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        # 调用CRFLayer进行解码\n",
    "        return self.crf.decode(embed, masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "178948b4",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-19, loss=47.09: 100%|█| 20/20 [04:04<00:00, 12.21s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAGwCAYAAABIC3rIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABXgElEQVR4nO3dd3SUZd7G8e9Meu+FQAKRHghFQAxgW0MRRETWgrEu6uoGEVAs74prR7ECsqDurt21ggVWBKkCoYUiTTokQAqQMimkzvP+ETMSaQGSzCRzfc6Zc5h5npn53bAm197VZBiGgYiIiIgTM9u7ABERERF7UyASERERp6dAJCIiIk5PgUhEREScngKRiIiIOD0FIhEREXF6CkQiIiLi9FztXUBjYLVaOXz4MH5+fphMJnuXIyIiIrVgGAYFBQVERUVhNp+5D0iBqBYOHz5MdHS0vcsQERGR85Cenk6LFi3OeI8CUS34+fkBVX+h/v7+dq5GREREasNisRAdHW37PX4mCkS1UD1M5u/vr0AkIiLSyNRmuosmVYuIiIjTUyASERERp6dAJCIiIk5Pc4hERESAyspKysvL7V2GnCN3d/ezLqmvDQUiERFxaoZhkJmZSV5enr1LkfNgNpuJjY3F3d39gj5HgUhERJxadRgKDw/H29tbG/A2ItUbJ2dkZBATE3NB/3YKRCIi4rQqKyttYSgkJMTe5ch5CAsL4/Dhw1RUVODm5nben6NJ1SIi4rSq5wx5e3vbuRI5X9VDZZWVlRf0OQpEIiLi9DRM1njV1b+dApGIiIg4PQUiERERcXoKRCIiIk6uVatWvPnmm3b/DHvSKjM7sloNcovLyC0uo0342U/iFRERAbjyyivp1q1bnQWQtWvX4uPjUyef1VgpENnRgZxirnp1Cd7uLmx7dpC9yxERkSbEMAwqKytxdT37r/qwsLAGqMixacjMjsL9PAAoLquksLTCztWIiAhUBYnisooGfxiGUav67rrrLpYuXcqUKVMwmUyYTCb279/PkiVLMJlM/PDDD/To0QMPDw+WL1/Onj17GDZsGBEREfj6+tKrVy9++umnGp/5x+Euk8nEv/71L4YPH463tzdt27blu+++O6e/x7S0NIYNG4avry/+/v7cdNNNZGVl2a5v2rSJq666Cj8/P/z9/enRowfr1q0D4MCBAwwdOpSgoCB8fHzo1KkT//vf/87p+8+VeojsyMfDFV8PVwpLK8iylOAb5mvvkkREnN7x8krinvqxwb9327MD8XY/+6/lKVOmsHPnTjp37syzzz4LVPXw7N+/H4DHH3+cV199lYsuuoigoCDS09MZPHgwL7zwAh4eHnz44YcMHTqUHTt2EBMTc9rveeaZZ5g8eTKvvPIK06ZNIykpiQMHDhAcHHzWGq1Wqy0MLV26lIqKCpKTk7n55ptZsmQJAElJSXTv3p0ZM2bg4uLCxo0bbRsrJicnU1ZWxrJly/Dx8WHbtm34+tbv70gFIjsL9/OgsLSCbEsprRWIRETkLAICAnB3d8fb25vIyMiTrj/77LP079/f9jw4OJiuXbvanj/33HPMnj2b7777jtGjR5/2e+666y5GjhwJwIsvvsjUqVNZs2YNgwadfYrHwoUL2bx5M/v27SM6OhqADz/8kE6dOrF27Vp69epFWloaEyZMoEOHDgC0bdvW9v60tDRGjBhBfHw8ABdddNFZv/NC2TUQLVu2jFdeeYXU1FQyMjKYPXs2119/ve26YRj84x//4N133yUvL4++ffsyY8aMGn9pOTk5PPjgg3z//feYzWZGjBjBlClTaiTJX375heTkZNauXUtYWBgPPvggjz76aEM29bTC/T3Ye7SI7IISe5ciIiKAl5sL254daJfvrQs9e/as8bywsJCnn36auXPnkpGRQUVFBcePHyctLe2Mn9OlSxfbn318fPD39yc7O7tWNWzfvp3o6GhbGAKIi4sjMDCQ7du306tXL8aPH88999zDRx99RGJiIjfeeCOtW7cGYMyYMTzwwAPMnz+fxMRERowYUaOe+mDXOURFRUV07dqV6dOnn/L65MmTmTp1KjNnzmT16tX4+PgwcOBASkp+Dw9JSUls3bqVBQsWMGfOHJYtW8Z9991nu26xWBgwYAAtW7YkNTWVV155haeffpp33nmn3ttXG+F+ngBkW0rtXImIiEDV/Blvd9cGf9TVjst/XC32yCOPMHv2bF588UV+/vlnNm7cSHx8PGVlZWf8nD+eC2YymbBarXVSI8DTTz/N1q1bGTJkCIsWLSIuLo7Zs2cDcM8997B3715uv/12Nm/eTM+ePZk2bVqdffep2DUQXXPNNTz//PMMHz78pGuGYfDmm2/y5JNPMmzYMLp06cKHH37I4cOH+eabb4CqBDpv3jz+9a9/0bt3b/r168e0adP47LPPOHz4MACffPIJZWVl/Oc//6FTp07ccsstjBkzhtdff70hm3paEf5VE6vVQyQiIrXl7u5e67O7VqxYwV133cXw4cOJj48nMjLSNt+ovnTs2JH09HTS09Ntr23bto28vDzi4uJsr7Vr145x48Yxf/58brjhBt577z3btejoaO6//35mzZrFww8/zLvvvluvNTvsKrN9+/aRmZlJYmKi7bWAgAB69+5NSkoKACkpKQQGBtboHkxMTMRsNrN69WrbPZdffrnt8DeAgQMHsmPHDnJzc0/53aWlpVgslhqP+lLdQ5SlHiIREamlVq1asXr1avbv38/Ro0fP2HPTtm1bZs2axcaNG9m0aRO33nprnfb0nEpiYiLx8fEkJSWxfv161qxZwx133MEVV1xBz549OX78OKNHj2bJkiUcOHCAFStWsHbtWjp27AjA2LFj+fHHH9m3bx/r169n8eLFtmv1xWEDUWZmJgARERE1Xo+IiLBdy8zMJDw8vMZ1V1dXgoODa9xzqs848Tv+aNKkSQQEBNgeJ46B1rVw9RCJiMg5euSRR3BxcSEuLo6wsLAzzgd6/fXXCQoKok+fPgwdOpSBAwdy8cUX12t9JpOJb7/9lqCgIC6//HISExO56KKL+PzzzwFwcXHh2LFj3HHHHbRr146bbrqJa665hmeeeQaoOrk+OTmZjh07MmjQINq1a8c///nPeq1Zq8xO4YknnmD8+PG25xaLpd5CkeYQiYjIuWrXrp1ttKRaq1atTrmXUatWrVi0aFGN15KTk2s8/+MQ2qk+Jy8v74w1/fEzYmJi+Pbbb095r7u7O//9739P+1n1PV/oVBy2h6h6KeGJmzhVP6++FhkZedKM94qKCnJycmrcc6rPOPE7/sjDwwN/f/8aj/ryew+RApGIiIi9OGwgio2NJTIykoULF9pes1gsrF69moSEBAASEhLIy8sjNTXVds+iRYuwWq307t3bds+yZcsoLy+33bNgwQLat29PUFBQA7Xm9CL8q3qICksrKNJu1SIiInZh10BUWFjIxo0b2bhxI1A1kXrjxo2kpaVhMpkYO3Yszz//PN999x2bN2/mjjvuICoqyrZXUfXY4r333suaNWtYsWIFo0eP5pZbbiEqKgqAW2+9FXd3d0aNGsXWrVv5/PPPmTJlSo0hMXvy9XDF271q7wn1EomIiNiHXecQrVu3jquuusr2vDqk3Hnnnbz//vs8+uijFBUVcd9995GXl0e/fv2YN28enp6etvd88sknjB49mquvvtq2MePUqVNt1wMCApg/fz7Jycn06NGD0NBQnnrqqRp7FdlbhL8n+44WkW0pITbUuU8bFhGxh9qeIyaOp67+7UyG/ldwVhaLhYCAAPLz8+tlPtFNb6ewZl8OU0d257quUXX++SIicmqVlZXs3LmT8PBwQkJC7F2OnIf8/HwOHz5MmzZtTtpM8lx+f2uVmQOoPvU+26Kl9yIiDcnFxYXAwEDbAh1vb+862zFa6p/VauXIkSN4e3vj6nphkUaByAFUT6zWHCIRkYZXveK4tud0iWMxm83ExMRccJBVIHIA6iESEbEfk8lEs2bNCA8Pr7EiWRoHd3d3zOYLXyOmQOQA1EMkImJ/Li4uuLjUzYnz0vg47D5EzqS6hyhLPUQiIiJ2oUDkAMLVQyQiImJXCkQOoPr4joKSCo6XVdq5GhEREeejQOQA/Dxc8XSr+qfQqfciIiINT4HIAZhMJk2sFhERsSMFIgehidUiIiL2o0DkIGwTqy3qIRIREWloCkQOwtZDpDlEIiIiDU6ByEGE+1X1EB1RD5GIiEiDUyByEBG/Lb3XpGoREZGGp0DkIKp7iDSpWkREpOEpEDmI6h4iBSIREZGGp0DkIMJ+m1RtKamgrMJq52pERESciwKRg/D3dMPVbAIgp6jMztWIiIg4FwUiB2E2mwj2cQfgaKEmVouIiDQkBSIHEuJbNWx2TD1EIiIiDUqByIGE+lb1EB1TD5GIiEiDUiByINVDZscK1UMkIiLSkBSIHEiIT9WQ2dEi9RCJiIg0JAUiBxLy25BZjnqIREREGpQCkQOxzSHSpGoREZEGpUDkQKqHzDSpWkREpGEpEDmQ6iGzoxoyExERaVAKRA4k1LYPUSmGYdi5GhEREeehQORAqnuISsqtFJdV2rkaERER56FA5EC83V3xdKv6J9FeRCIiIg1HgcjBaC8iERGRhqdA5GB+P75DPUQiIiINRYHIwVQf8JqjHiIREZEGo0DkYEJ8tPReRESkoSkQOZjqHiINmYmIiDQcBSIH8/vxHRoyExERaSgKRA4mRJOqRUREGpwCkYOxLbvXeWYiIiINRoHIwQT76MR7ERGRhqZA5GBCbcvuy7BadZ6ZiIhIQ1AgcjDVPUSVVgNLSbmdqxEREXEOCkQOxt3VjL+nK6C9iERERBqKApEDCrXtRaSJ1SIiIg1BgcgB2Zbea2K1iIhIg1AgckDVS+/VQyQiItIwFIgcUHUPkeYQiYiINAwFIgcU4qPjO0RERBqSApED0gGvIiIiDUuByAHpPDMREZGGpUDkgGyTqjVkJiIi0iAUiBxQqJbdi4iINCgFIgdUPYcor7ic8kqrnasRERFp+hSIHFCglxtmU9Wfc9VLJCIiUu8UiByQ2Wwi+Ld5RNqLSEREpP4pEDmo3+cRaWK1iIhIfVMgclDBPlp6LyIi0lAUiBxU9cTqozrPTEREpN4pEDmo6uM7cjSpWkREpN45dCCqrKxk4sSJxMbG4uXlRevWrXnuuecwDMN2j2EYPPXUUzRr1gwvLy8SExPZtWtXjc/JyckhKSkJf39/AgMDGTVqFIWFhQ3dnHMSqt2qRUREGoxDB6KXX36ZGTNm8NZbb7F9+3ZefvllJk+ezLRp02z3TJ48malTpzJz5kxWr16Nj48PAwcOpKSkxHZPUlISW7duZcGCBcyZM4dly5Zx33332aNJtWY7z0yTqkVEROqdq70LOJOVK1cybNgwhgwZAkCrVq3473//y5o1a4Cq3qE333yTJ598kmHDhgHw4YcfEhERwTfffMMtt9zC9u3bmTdvHmvXrqVnz54ATJs2jcGDB/Pqq68SFRVln8adRfWQmZbdi4iI1D+H7iHq06cPCxcuZOfOnQBs2rSJ5cuXc8011wCwb98+MjMzSUxMtL0nICCA3r17k5KSAkBKSgqBgYG2MASQmJiI2Wxm9erVp/ze0tJSLBZLjUdDUw+RiIhIw3HoHqLHH38ci8VChw4dcHFxobKykhdeeIGkpCQAMjMzAYiIiKjxvoiICNu1zMxMwsPDa1x3dXUlODjYds8fTZo0iWeeeaaum3NONIdIRESk4Th0D9EXX3zBJ598wqeffsr69ev54IMPePXVV/nggw/q9XufeOIJ8vPzbY/09PR6/b5Tqd6HqLiskuKyigb/fhEREWfi0D1EEyZM4PHHH+eWW24BID4+ngMHDjBp0iTuvPNOIiMjAcjKyqJZs2a292VlZdGtWzcAIiMjyc7OrvG5FRUV5OTk2N7/Rx4eHnh4eNRDi2rP18MVd1czZRVWjhWW4R3s0P9UIiIijZpD9xAVFxdjNtcs0cXFBau16gT42NhYIiMjWbhwoe26xWJh9erVJCQkAJCQkEBeXh6pqam2exYtWoTVaqV3794N0IrzYzKZCK3erVp7EYmIiNQrh+52GDp0KC+88AIxMTF06tSJDRs28Prrr/OXv/wFqAoNY8eO5fnnn6dt27bExsYyceJEoqKiuP766wHo2LEjgwYN4t5772XmzJmUl5czevRobrnlFoddYVYtxNeDw/kl5GhitYiISL1y6EA0bdo0Jk6cyN/+9jeys7OJiorir3/9K0899ZTtnkcffZSioiLuu+8+8vLy6NevH/PmzcPT09N2zyeffMLo0aO5+uqrMZvNjBgxgqlTp9qjSeckxFdL70VERBqCyThx22c5JYvFQkBAAPn5+fj7+zfY9z78xSa+Xn+QxwZ14IErWzfY94qIiDQF5/L726HnEDm735fea8hMRESkPikQObDqITNNqhYREalfCkQOLMSnaun/UfUQiYiI1CsFIgcWrN2qRUREGoQCkQML9dF5ZiIiIg1BgciBVc8hyikqQ4sBRURE6o8CkQOrPs+svNLAUqLzzEREROqLApED83Rzwc+jau9MLb0XERGpPwpEDk5L70VEROqfApGDC/H9bWK1eohERETqjQKRgwvx0XlmIiIi9U2ByMH93kOkQCQiIlJfFIgcXHUPkfYiEhERqT8KRA4uRLtVi4iI1DsFIgdnGzJTD5GIiEi9USBycKE+6iESERGpbwpEDq66h0gn3ouIiNQfBSIHF/rbHKLc4nLKK612rkZERKRpUiBycEHe7riaTYCGzUREROqLApGDM5tNhP42bJZdUGLnakRERJomBaJGIMyvKhAdKdA8IhERkfqgQNQIKBCJiIjULwWiRiDcr3rITIFIRESkPigQNQLqIRIREalfCkSNwO89RJpULSIiUh8UiBoB9RCJiIjULwWiRsAWiLRbtYiISL1QIGoEwv08Aci2lGIYhp2rERERaXoUiBqB6h6i0gorBaUVdq5GRESk6VEgagQ83Vzw83QFNI9IRESkPigQNRLVvUTZFgUiERGRuqZA1EiE+WpitYiISH1RIGokwv2rJ1ZrLyIREZG6pkDUSKiHSEREpP4oEDUS2pxRRESk/igQNRLhCkQiIiL1RoGokVAPkYiISP1RIGokwv0ViEREROqLAlEjUT2p+lhRGeWVVjtXIyIi0rQoEDUSQd7uuJpNABwrLLNzNSIiIk2LAlEjYTabCP2tlyi7QHsRiYiI1CUFokZEE6tFRETqhwJRI6JAJCIiUj8UiBqR6r2IshWIRERE6pQCUSOiHiIREZH6oUDUiPzeQ6RJ1SIiInVJgagRUQ+RiIhI/VAgakRsgUgn3ouIiNQpBaJGJMzXE1APkYiISF1TIGpEQnzdASgpt1JcVmHnakRERJoOBaJGxNvdBU+3qn8yHd8hIiJSdxSIGhGTyUSIT9U8oqOaRyQiIlJnFIgamdDfhs3UQyQiIlJ3FIgamZDfDng9VqQeIhERkbqiQNTIBPtU9RAdVQ+RiIhInVEgamRCNGQmIiJS5xSIGplQHw2ZiYiI1DUFokamuocop0g9RCIiInXF4QPRoUOHuO222wgJCcHLy4v4+HjWrVtnu24YBk899RTNmjXDy8uLxMREdu3aVeMzcnJySEpKwt/fn8DAQEaNGkVhYWFDN6VOVE+q1hwiERGRuuPQgSg3N5e+ffvi5ubGDz/8wLZt23jttdcICgqy3TN58mSmTp3KzJkzWb16NT4+PgwcOJCSkt9PhE9KSmLr1q0sWLCAOXPmsGzZMu677z57NOmChfhUzyHSkJmIiEhdMRmGYdi7iNN5/PHHWbFiBT///PMprxuGQVRUFA8//DCPPPIIAPn5+URERPD+++9zyy23sH37duLi4li7di09e/YEYN68eQwePJiDBw8SFRV10ueWlpZSWvp74LBYLERHR5Ofn4+/v389tLT2MvNLuHTSQlzNJnY+fw1ms8mu9YiIiDgqi8VCQEBArX5/O3QP0XfffUfPnj258cYbCQ8Pp3v37rz77ru26/v27SMzM5PExETbawEBAfTu3ZuUlBQAUlJSCAwMtIUhgMTERMxmM6tXrz7l906aNImAgADbIzo6up5aeO6CfNwAqLAaWErK7VyNiIhI0+DQgWjv3r3MmDGDtm3b8uOPP/LAAw8wZswYPvjgAwAyMzMBiIiIqPG+iIgI27XMzEzCw8NrXHd1dSU4ONh2zx898cQT5Ofn2x7p6el13bTz5uHqgp+nK6B5RCIiInXF1d4FnInVaqVnz568+OKLAHTv3p0tW7Ywc+ZM7rzzznr7Xg8PDzw8POrt8y9UqK8HBSUVHCsspU24r73LERERafQcuoeoWbNmxMXF1XitY8eOpKWlARAZGQlAVlZWjXuysrJs1yIjI8nOzq5xvaKigpycHNs9jY1tYrWW3ouIiNQJhw5Effv2ZceOHTVe27lzJy1btgQgNjaWyMhIFi5caLtusVhYvXo1CQkJACQkJJCXl0dqaqrtnkWLFmG1Wundu3cDtKLu2XarViASERGpEw49ZDZu3Dj69OnDiy++yE033cSaNWt45513eOeddwAwmUyMHTuW559/nrZt2xIbG8vEiROJiori+uuvB6p6lAYNGsS9997LzJkzKS8vZ/To0dxyyy2nXGHWGNgOeNXSexERkTrh0IGoV69ezJ49myeeeIJnn32W2NhY3nzzTZKSkmz3PProoxQVFXHfffeRl5dHv379mDdvHp6enrZ7PvnkE0aPHs3VV1+N2WxmxIgRTJ061R5NqhOhPjrPTEREpC459D5EjuJc9jFoCO+v2MfT329jcHwk/0zqYe9yREREHFKT2YdITk3Hd4iIiNQtBaJGyDapWnOIRERE6oQCUSMUWj2pWqvMRERE6oQCUSNUvQ9RXnE5FZVWO1cjIiLS+CkQNUKB3u5Un+maU6xeIhERkQulQNQIuZhNBGvpvYiISJ1RIGqkFIhERETqznkFog8++IC5c+fanj/66KMEBgbSp08fDhw4UGfFyemF+FRPrNZKMxERkQt1XoHoxRdfxMvLC4CUlBSmT5/O5MmTCQ0NZdy4cXVaoJxa9dJ77UUkIiJy4c7r6I709HTatGkDwDfffMOIESO477776Nu3L1deeWVd1ienEarzzEREROrMefUQ+fr6cuzYMQDmz59P//79AfD09OT48eN1V52cVojmEImIiNSZ8+oh6t+/P/fccw/du3dn586dDB48GICtW7fSqlWruqxPTiNEmzOKiIjUmfPqIZo+fToJCQkcOXKEr7/+mpCQEABSU1MZOXJknRYop2ZbZaZJ1SIiIhfsvHqIAgMDeeutt056/ZlnnrnggqR2Qn01ZCYiIlJXzquHaN68eSxfvtz2fPr06XTr1o1bb72V3NzcOitOTi/C3xOAzPwSHd8hIiJygc4rEE2YMAGLxQLA5s2befjhhxk8eDD79u1j/PjxdVqgnFrzQC883cyUVVpJyym2dzkiIiKN2nkNme3bt4+4uDgAvv76a6699lpefPFF1q9fb5tgLfXLbDbRJtyXLYcs7Mwq5KIwX3uXJCIi0midVw+Ru7s7xcVVvRI//fQTAwYMACA4ONjWcyT1r124HwC7swvsXImIiEjjdl49RP369WP8+PH07duXNWvW8PnnnwOwc+dOWrRoUacFyum1iajqFdqZVWjnSkRERBq38+oheuutt3B1deWrr75ixowZNG/eHIAffviBQYMG1WmBcnrVPUS7shWIRERELsR59RDFxMQwZ86ck15/4403Lrggqb22v/UQ7TlSSKXVwMVssnNFIiIijdN5BSKAyspKvvnmG7Zv3w5Ap06duO6663Bxcamz4uTMWgR54+lmpqS8aqVZbKiPvUsSERFplM4rEO3evZvBgwdz6NAh2rdvD8CkSZOIjo5m7ty5tG7duk6LlFNzMZtoHebL1sMWdmUVKBCJiIicp/OaQzRmzBhat25Neno669evZ/369aSlpREbG8uYMWPqukY5g3YRmkckIiJyoc6rh2jp0qWsWrWK4OBg22shISG89NJL9O3bt86Kk7NrE141j2hXlpbei4iInK/z6iHy8PCgoODkX8CFhYW4u7tfcFFSe9U9RFp6LyIicv7OKxBde+213HfffaxevRrDMDAMg1WrVnH//fdz3XXX1XWNcgZtw2uuNBMREZFzd16BaOrUqbRu3ZqEhAQ8PT3x9PSkT58+tGnThjfffLOOS5QziQ72xsPVTGmFlYO5OtNMRETkfJzXHKLAwEC+/fZbdu/ebVt237FjR9q0aVOnxcnZVa8025ZRdaZZyxCtNBMRETlXtQ5EZzvFfvHixbY/v/766+dfkZyzdhFVgWhXdgH94yLsXY6IiEijU+tAtGHDhlrdZzJpt+SG1rZ66b0mVouIiJyXWgeiE3uAxLHYlt7r1HsREZHzcl6TqsWxVAeivUeKMAytNBMRETlXCkRNQIsgL0wmKC6r5Ghhmb3LERERaXQUiJoAD1cXogK8AEjL0dJ7ERGRc6VA1ETEBHsDkJZTZOdKREREGh8FoiaiOhAdOKYeIhERkXOlQNRExIRU9xApEImIiJwrBaImwjZkph4iERGRc6ZA1ES0VA+RiIjIeVMgaiJaBledYZZdUMrxsko7VyMiItK4KBA1EQHebvh7Vm08rl4iERGRc6NA1IRUn3SvQCQiInJuFIiakN+X3msvIhERkXOhQNSEVC+9T1cPkYiIyDlRIGpCbD1ECkQiIiLnRIGoCWmpvYhERETOiwJRE1I9ZHYw9ziVVsPO1YiIiDQeCkRNSLMAL9xcTJRVWsm0lNi7HBERkUZDgagJcTGbaBGklWYiIiLnSoGoiYkO1kozERGRc6VA1MS0tO1FpEAkIiJSWwpETYwOeRURETl3CkRNTPWQmQKRiIhI7SkQNTEXhVadZ7Ynu1BL70VERGpJgaiJuSjMF293F4rKKtmdXWjvckRERBoFBaImxsVsIr55AAAb03PtXI2IiEjjoEDUBHWLCQRgY3q+fQsRERFpJBpVIHrppZcwmUyMHTvW9lpJSQnJycmEhITg6+vLiBEjyMrKqvG+tLQ0hgwZgre3N+Hh4UyYMIGKiooGrr7hdI8OBGBjep5d6xAREWksGk0gWrt2LW+//TZdunSp8fq4ceP4/vvv+fLLL1m6dCmHDx/mhhtusF2vrKxkyJAhlJWVsXLlSj744APef/99nnrqqYZuQoPpFh0EwI5MC8VlTTf4iYiI1JVGEYgKCwtJSkri3XffJSgoyPZ6fn4+//73v3n99df505/+RI8ePXjvvfdYuXIlq1atAmD+/Pls27aNjz/+mG7dunHNNdfw3HPPMX36dMrKyk75faWlpVgslhqPxiQywJMIfw+sBmw51LhqFxERsYdGEYiSk5MZMmQIiYmJNV5PTU2lvLy8xusdOnQgJiaGlJQUAFJSUoiPjyciIsJ2z8CBA7FYLGzduvWU3zdp0iQCAgJsj+jo6HpoVf3qZhs208RqERGRs3H4QPTZZ5+xfv16Jk2adNK1zMxM3N3dCQwMrPF6REQEmZmZtntODEPV16uvncoTTzxBfn6+7ZGenl4HLWlY1cNmmkckIiJydq72LuBM0tPTeeihh1iwYAGenp4N9r0eHh54eHg02PfVh+oeok1aaSYiInJWDt1DlJqaSnZ2NhdffDGurq64urqydOlSpk6diqurKxEREZSVlZGXl1fjfVlZWURGRgIQGRl50qqz6ufV9zRF8S0CMJngUN5xsgtK7F2OiIiIQ3PoQHT11VezefNmNm7caHv07NmTpKQk25/d3NxYuHCh7T07duwgLS2NhIQEABISEti8eTPZ2dm2exYsWIC/vz9xcXEN3qaG4uvhSrtwPwA2puXZtxgREREH59BDZn5+fnTu3LnGaz4+PoSEhNheHzVqFOPHjyc4OBh/f38efPBBEhISuPTSSwEYMGAAcXFx3H777UyePJnMzEyefPJJkpOTG/2w2Nl0iw5kR1YBmw7mMaBT0+0NExERuVAO3UNUG2+88QbXXnstI0aM4PLLLycyMpJZs2bZrru4uDBnzhxcXFxISEjgtttu44477uDZZ5+1Y9UN4/cdq/PsWoeIiIijMxmGoSPRz8JisRAQEEB+fj7+/v72LqfWdmQWMPDNZXi4mtnwVH+83R26Q1BERKROncvv70bfQySn1y7Cl5hgb0orrCzZccTe5YiIiDgsBaImzGQycU3nqrlD/9ucYedqREREHJcCURN3TXwzABb/mk1JeaWdqxEREXFMCkRNXNcWAUQFeFJUVsnPu47auxwRERGHpEDUxJlMJgb+Nmz2g4bNRERETkmByAkM/m3YbMH2LMoqrHauRkRExPEoEDmBHjFBhPl5UFBSwco9GjYTERH5IwUiJ2A2mxjYKQKAHzZn2rkaERERx6NA5CQGd64aNpvzy2GOFZbauRoRERHHokDkJC69KIT45gEUlVUyffEee5cjIiLiUBSInITZbGLCwPYAfLzqAIfyjtu5IhEREcehQORELmsbSsJFIZRVWnlzwU57lyMiIuIwFIiciMlkYsKgql6ir9cfZHd2gZ0rEhERcQwKRE7m4pggBsRFYDXgjQW77F2OiIiIQ1AgckJjrm4LwE/bsyit0PlmIiIiCkROqFOUP2F+HpRWWNmQlmfvckREROxOgcgJmUwmEi4KAWDlnmN2rkZERMT+FIicVJ/WVYEoRUd5iIiIKBA5qz6tQwHYkJZHcVmFnasRERGxLwUiJxUd7EXzQC8qrAZr9+fauxwRERG7UiByUiaTyTZstlLDZiIi4uQUiJxYnzZVgWiVJlaLiIiTUyByYgkXVc0j2nwon/zj5XauRkRExH4UiJxYZIAnF4X5YDVgzb4ce5cjIiJiNwpETk7ziERERBSInF718vulO45gGIadqxEREbEPBSInd1nbULzcXNh7tIj1OsZDRESclAKRk/PzdGNwfDMAvlibbudqRERE7EOBSLi5VzQAc345TFGpdq0WERHno0Ak9GoVxEWhPhSVVTL3lwx7lyMiItLgFIgEk8nEjT2reok+X6dhMxERcT4KRALAiB7NcTGbSD2Qy+7sAnuXIyIi0qAUiASAcD9P/tQhHIDPNblaREScjAKR2Nzy2+TqD1MOsDNLvUQiIuI8FIjE5qr24VzRLozSCisPfrqBkvJKe5ckIiLSIBSIxMZsNvHqjV0J9fVgR1YBL/5vu71LEhERaRAKRFJDmJ8Hr9/UFagaOvvP8n0cKyy1c1UiIiL1y2ToAKuzslgsBAQEkJ+fj7+/v73LaRAv/m877yzba3veKcqfpN4tGXlJNCaTyY6ViYiI1M65/P5WD5Gc0iMD2jO+fzs6Nqv6H9DWwxb+b/Zm7vjPGg7nHbdzdSIiInVLPUS14Iw9RCc6UlDK7A0HeW3+TkorrPh5uDJlZDf+1CHC3qWJiIiclnqIpE6F+Xlw3+Wt+d9Dl9EtOpCC0goe/3oz5ZVWe5cmIiJSJxSIpNZah/nyxV8TCPX1ILuglAXbsuxdkoiISJ1QIJJz4u5qtm3g+FHKAdvrVqvBj1szycwvsVdpIiIi502BSM7ZyN4xmE2QsveY7dyzqYt28dePUrn13VXa0FFERBodBSI5Z80DvWwTqj9elcbKPUeZsnAXAHuPFvHGTzvtWZ6IiMg5UyCS83LbpTEAfL3+IA99thHDgG7RgQC8u2wvG9Pz7FeciIjIOVIgkvNyedswYoK9KSip4EhBKW3DffnvvZcyrFsUVgMmfLmJ1AM5vDB3G0OnLeer1IP2LllEROS0FIjkvJjNJpJ6V/USebqZmZ50MV7uLjw9tBOhvu7syi5kxIwU3v15H5sP5fPa/B1oyysREXFUCkRy3u5IaMVf+sby7h09aRfhB0CQjzsvDI/HbAJvdxeGdo3C292FjPwSfjmYb+eKRURETk07VdeCs+9UfT4y80sI8HLDy92F5E/XM/eXDB64sjWPDepg79JERMRJaKdqsbvIAE+83F0AGNQpEoB5WzI1bCYiIg5JgUjq3VUdwnF3MbPvaBG7sgvtXY6IiMhJFIik3vl6uHJZ21CgqpdIRETE0SgQSYMY2Pn3YTMRERFHo0AkDSKxYwQuZhPbMiykHSu2dzkiIiI1KBBJgwj2cad3bDAA87Zm2LkaERGRmhSIpMFc89uw2dtL93I477idqxEREfmdQweiSZMm0atXL/z8/AgPD+f6669nx44dNe4pKSkhOTmZkJAQfH19GTFiBFlZWTXuSUtLY8iQIXh7exMeHs6ECROoqKhoyKYIcGPPaOKa+XOsqIy/fpRKSXmlvUsSEREBHDwQLV26lOTkZFatWsWCBQsoLy9nwIABFBUV2e4ZN24c33//PV9++SVLly7l8OHD3HDDDbbrlZWVDBkyhLKyMlauXMkHH3zA+++/z1NPPWWPJjk1TzcX3r69B0Hebmw+lM//zdqsfYlERMQhNKqdqo8cOUJ4eDhLly7l8ssvJz8/n7CwMD799FP+/Oc/A/Drr7/SsWNHUlJSuPTSS/nhhx+49tprOXz4MBEREQDMnDmTxx57jCNHjuDu7n7W79VO1XVr5e6j3P6fNVRaDcb3b8eDf2qDyWSyd1kiItLENNmdqvPzq87CCg6umpybmppKeXk5iYmJtns6dOhATEwMKSkpAKSkpBAfH28LQwADBw7EYrGwdevWU35PaWkpFoulxkPqTp82ofzf4I4AvL5gJ3//ZgsVlVY7VyUiIs6s0QQiq9XK2LFj6du3L507dwYgMzMTd3d3AgMDa9wbERFBZmam7Z4Tw1D19eprpzJp0iQCAgJsj+jo6DpujfylbysmXhuHyQSfrk5j1AfrKCgpt3dZIiLipBpNIEpOTmbLli189tln9f5dTzzxBPn5+bZHenp6vX+nszGZTIzqF8vbt/XAy82FpTuPcNu/15wUihrRiK6IiDRijSIQjR49mjlz5rB48WJatGhhez0yMpKysjLy8vJq3J+VlUVkZKTtnj+uOqt+Xn3PH3l4eODv71/jIfVjQKdIPv/rpQR5u7EpPY+73ltLUWkFhaUVPDdnG53+8SPvrdhX4z05RWU88/1WdmUV2KlqERFpahw6EBmGwejRo5k9ezaLFi0iNja2xvUePXrg5ubGwoULba/t2LGDtLQ0EhISAEhISGDz5s1kZ2fb7lmwYAH+/v7ExcU1TEPkjLq0COSjUb3x93Ql9UAut767isTXlvLv5fsoLqtk2qLdlFb8vkT/tfk7eG/FfkZ/uoFKq3qQRETkwjl0IEpOTubjjz/m008/xc/Pj8zMTDIzMzl+vGpTv4CAAEaNGsX48eNZvHgxqamp3H333SQkJHDppZcCMGDAAOLi4rj99tvZtGkTP/74I08++STJycl4eHjYs3lygs7NA/hoVG/8PFzZdDCfTEsJMcHehPq6k1NUxoJtVb16lpJyZm84BMCOrAJmrT9oz7JFRKSJcOhANGPGDPLz87nyyitp1qyZ7fH555/b7nnjjTe49tprGTFiBJdffjmRkZHMmjXLdt3FxYU5c+bg4uJCQkICt912G3fccQfPPvusPZokZ9A1OpAPRl1Cz5ZBjLm6LfPHXc6tl8QA8N81aQB8nXqQ4rJKXM1Vy/RfX7BTGzyKiMgFa1T7ENmL9iGyn4O5xVw2eTGGAYsfuZJR769l79EinhzSkfdW7OdQ3nEeG9SBB65sbe9SRUTEwTTZfYjE+bQI8uaKdmEAjP9iI3uPFuHn4crIS2IY378dAP9cspvcojJ7likiIo2cApE4vJG/DZttSMsDYESPFvh4uHJ99+Z0bOZPQUkFz83dpiX6IiJy3hSIxOH9qUM4YX6/T4C/PaElAC5mE09dG4fZBLPWH+LtZXvtVaKIiDRyCkTi8NxczNzcs2q38MvahtI6zNd2LaF1CE9dW7V9wsvzfmXelqrdx7MtJaQeyNGEaxERqRVXexcgUhuj/9SGYB93hnRpdtK1u/rGsvdoER+mHGDs5xsI+58H6TlVWzO0DPHmxeHx9G0TSmFpBZ+sOsDSnUeYMLA93WOCGroZIiLioLTKrBa0yszxVVRaGfXBOpbuPAKAyQQ+7q4UllYAcGX7MNYfyMVSUvW8S4sAvk3ui8lkslvNIiJSv87l97d6iKRJcHUxM+O2i/lu42FaBHnTNToAgFd+3MFHqw6wZEdVULoozIdDucf55WA+q/bmkNA6xJ5li4iIg1APUS2oh6hxSz2Qyxdr07myfRgDOkXy9Hdb+WjVAa5sH8b7d19i7/JERKSeaB8ikRP0aBnEy3/uwjXxzXAxm7jnsljMJliy4wi/ZlpOun/h9iyunfYzqQdy7VCtiIjYgwKROJ2WIT5c07lqcvY7f1iqvz3DwuhPN7DlkIXX5u+wR3kiImIHCkTilO67/CIAvtt4mIO5xQDkFZdx30frOP7bUv2Ve46x72iR3WoUEZGGo0AkTqlrdCCXXhRMhdWg/+vLmPDlJh74eD3pOceJDvbikthg4PdDZQGKyyqYtyWDY4WlNT5rz5FCvko9SHFZRYO2QURE6o4mVdeCJlU3TTsyC/jbJ6nsOfJ7L5CXmwuz/taHg7nHuffDdQR5u5HyxNW4uZi5/d+rWbnnGG4uJgbERdK3TShzfjnMyj3HAOjYzJ937+hBiyBvezVJREROcC6/vxWIakGBqOkyDIPUA7l8tjadtftz+L/BHRnYKZKKSiv9Xl5MpqWEKbd0Y3d2IdMW7cZsAusf/osx/7bnUUFpBaG+7rx9ew96tAy2T4NERMRGgaiOKRA5pzd/2smbP+2ieaAXh/Kqdr6ecks32oT78vnadDal53FZ2zBG9q46fPbeD9axLcOCu4uZj0ZdQu+LtMeRiIg9KRDVMQUi55SRf5y+Ly2y9QjddmkMz18ff9r7i8sqePDTDSz8NZtu0YHM/lsf207YZRVWtmVY6Bzlj6uLpu6JiDQE7UMkUgeaBXhxdccIAOKbBzDxt0NkT8fb3ZWXRnTB083MxvQ82zEihmHw14/Wcf30FSS+vpQv1qVTXmmt9/pFRKT2FIhEzuAfQ+MYfVUb/nVnTzxcXc56f5ifB7df2hKAN3/ahWEYfLzqAIt/Ozpk/7FiHv3qF656dQkb0mpu/Gi1GietYAPItpSwfNdRrH+cvCQiInVGQ2a1oCEzORdHCkq5bPIiSsqtPHVtHJN//JWScisTBrbHzcXEO8v2cbSwFG93F96+vQeXtQ1j75FCxn+xiY3peVzeLowHrmhNXDN/Zizdw3sr9lFaYWV49+ZM/nMX3M4y5PZrpoX1B/K4pVc0ZrMOrxUR56U5RHVMgUjO1Qtzt/Huz/tsz/u2CeGjv/TGbDZRVFrB/R+n8vOuo7i5mEjq3ZLP1qZRUl5zGM3dxUzZH4bWru4QzvSki/F0O3Vv1e7sQob/cwUFJRW8dmNXRvRoUfeNExFpJBSI6pgCkZyrE3uJ/D1d+XHc5TQL8LJdL62oZPznm5i7OcP2Wp/WIYzv345vNx7mi3XplFZYaR/hx6OD2gPwt0/WU1phpUOkHyG+7mTml+BqNjPm6rYMjo8kt7ic66evIC2nauftbtGBfJPct2EbLiLiQBSI6pgCkZyPaQt3MWXhLqaO7M7g+GYnXa+0Gjw/dxvfbTxM8lVtuKtPK9sQ19HCUg7lHqdz8wBcfnttzb4cRr2/loLSk3fE7h8XQW5RGesO5NI80IvsghLKKw3mPNiPzs0DAJi3JYPlu4/yyID2BHq712PLRUQcgwJRHVMgkvNVXmk965yfc3HgWBELt2cT6O1GpL8nq/YeY8bSPZRXVv1n7Ofpyuy/9eHNn3Yx55cMRl4SzaQburA9w8LQacupsBr0bBnEx/f0Pu2wm4hIU6Fl9yIOoi7DEEDLEB/+0i+WGy5uQZ82oYwf0J45D15Gj5ZB+Hq4MiOpB23C/bjtt5Vu32w4TG5RGRO+2kTFb6vU1h3I5eEvNtV61VpecRmjP13P8H+u4EjByavgRESaAvUQ1YJ6iKQxOLE3yjAMBryxjF3ZhXRs5s/2DAsBXm48O6wTj3y5ifJKg1H9YnlySEfb5pHVKq2GbZhuy6F87v84lYO5VTt1D4lvxvSki8+7xhf/t51Decd57cau6qESkXp3Lr+/XRuoJhGpZyf2RplMJpJ6x/D099vYnmEB4JnrOjGsW3MAHvpsI/9evo99R4uYdEM8Ef6erNxzlBfmbmdbhoWoAC9igr1JTculrMJKiyAvMvJLmLs5g6FbMhnUueq8t+mL9/DzriNYDQOrAbGhPrw0Iv6UezZtPZzPO8v2AhDk7XbGXb9FRBqaApFIE3VDjxa8PG8Hx8srSewYwbBuUQAM69YcS0kFz32/jUW/ZtP/9aV0jQ7k511Hbe89lHfcdn5bYsdwXrupG+8s28P0xXuY+O0WOkT68fdvNrNi97Ea37kxPY/oYG/G9293Uj0frzpwwp/T6NcmlEGdT55sLiJiDxoyqwUNmUlj9VHKfhZsz+bVP3ch3N+zxrUdmQVM+GoTvxzMB8DFbOK23jHc3TeWY0Wl7DtajJ+nK/07RmA2mygpr2TI1J/Zc6QIV7OJCquBt7sLjw3qQLMAT/YcKeLleb/i5mLif2Muo22En+278o+Xc+mLCzleXsllbUP5eddR/D1d+WHs5TQP9EJEpD5olVkdUyCSpqqi0sp7K/bza2YBD1x5EW3C/c54/7r9Odz4dgqGAdHBXrx7R086RFb9N2EYBvd+uI6ftmfTs2UQX/w1wbaNwH+W7+PZOdtoH+HH9w/248aZK9l0MJ/WYT5c2T6c2FAf2kX40bm5P97u599xvXzXUcxm6NM69Lw/Q0SaDgWiOqZAJPK7L9als+2whbGJbU/az+hw3nH6v76UorJKnr++M7dd2hLDMLj6taXsPVrEc9d35vZLW3LgWBHXTl1+0p5KLmYTHSL9CPZxJ8tSQmZ+CcE+7gzv3oI/92xB80AvDMOgoLQCH3dX2+RvgP+uSeOJWZsBGN69OU8P7USAt1v9/4WIiMNSIKpjCkQitffein088/02PN3M3NUnlg6Rfoz9fCO+Hq6s+r+r8fWo6gE6lHecRb9ms/9oEfuOFrH1cD5ZltMv6zeZINzPg9yicsoqrYT6evD3IR24vltz5vySwZjPNnDiT7MIfw+eurYT/eMicHc1YxgGP27N5O1le2kR5M1rN3bF3fX3ieh7jxTi7e5KZIDnKb5dRBojBaI6pkAkUnuVVoO73ltTY5I2wB0JLXl2WOczvjcj/zgb0vIoKq0gMsCTCH9Pth228PnadFL2Hjvle7pFB7LlUD4VVoOk3jGM6NGCR77YxN6jRQAEeLlxTedIthzOZ8shi+19N/eM5qUR8ZhMJmatP8gjX27C1Wxm1GWxjL6qDT4eWnMi0tgpENUxBSKRc2O1Giz6NZt/LtnN+rQ8XMwm5j1Uc6L1uTqUd5yjBaWE+nng5+nKRykHmLZol+1Q3GHdonjjpm6YzSaOl1Xy1uJdfLnuINknbCbp4+7CtV2i+DI1HasBj1/TgQAvN/5v9uYavUvhfh70j4vgeHklpeVW+rUN5ZZe0Sft2bQ7u5AvU9OZsymD7jGBvHZT11NuOSAi9qFAVMcUiETOj2EYbEzPw2Qy0S06sM4/Pz2nmDcW7MTX05WJ18adtDN4pdVg9d5jzNuaSaC3O3f1aUWwj7ttWO9Et1/aksvahvL83O22A3JPdHPPaJ4f3hk3FzOpB3J5ed6vrNmXU+OexI7h/DOpR42hOIDFv2Yz+ccdVFqttI/0p0OkH4kdI2gfef4B8Vxl5B9n+uLdXNsliksvCmmw7xWxJwWiOqZAJNK0GIbBP77byocpVXsj3dMvlr//tmt3aUUlX6UeJMtSire7C3nF5byzbA9WA/q2CSHU14NvNx4GqiaBX9U+jN6xIbw6fwelFVYGxEUwPeli3FzM5B8v57k52/gq9eAp6xgS34yHEtvS7oSes0qrwYcp+3lr0W6shkGLIG+ig724u28svVoF2+4rq7DyzYZDJLQOITrY+4ztLSqt4M8zU9ieYcHD1cyHf7mE3gpF4gQUiOqYApFI01O903aonzu3XhJz0nDYiRb9msXoTzdQXFYJVE3wvqlHNOP6t7NNwl628wj3fLiOsgornm5mXM1myiqtlFVYMZlgVN9Y+rQJYUdmIakHcvhpe7btsy5pFUyf1qG0j/Rl+uI9bD6Uf1INXm4ufHl/Ap2bB2AYBuM+38g3Gw/TOsyHeWMvP+25eVarwd8+Wc+8rZm21/w8XPnvfZfSuXnAef/9iTQGCkR1TIFIRLYezmfsZxsJ9/fgiWs6njJMLNmRTfIn6yn6LTgBtArx5tUbu9LzhN4dgF8zLUz5aRc/bMn848fg5+nKY4M60KNlEAdzj/P+yn2s2H2MZgGefJvclw9TDvDW4t22+5+6No6/9IsFYP/RIp7+fiuhvh5c0iqY3UcKeWfZXtxdzLx/dy+mLNzF6n05hPi488X9CbQO87V9TnpOMa/N38H+Y8UcKSilsLSCfm1DGdUvlotjgs76d3S0sJQQH/czhkuRhqRAVMcUiESktopKKzhaWGqbpN0iyAvX0/TeABw4VsTPu46SsvcYWw7lc3FMEE8M7kC43+/L/y0l5QyfvoI9R4poHuh1wrEqEfy0PQt/T1eWTLgKgBv+uYL9x06eA/XKn7twY89oCkrKGfnuKrYcshAV4MlXD/QhKtCLnKKy074Xqlbz3dizBYM6RRLi61HjWnFZBU/O3sKsDYeIDfXhhu7NGX5xc1oEnXko748Mw2DzoXw2pudxfffm+Hs2nX2k9h4p5MCxYq7qEG7vUpyKAlEdUyASEXvbf7SI6/+5grzicgDG/KkNDyW2Y+i05WzLsHBTzxbsO1rE2v25NA/04rpuUazbn8PWwxZG9Yvl4QHtbZ91rLCUG99OYe+RIlqH+fDRqN48+N8NpB6oeu/Ea+MI9/fAMOCzNWl8u/EwZZVVq/nMJrj0ohD+1CGcy9qG4WI28bdPUtmZVVijXpMJ7ri0JU8M7oinW82Vd4ZhkH+8nMN5JRSUlFNUVsHeI0V8lXqQXzMLAOgU5c/Ho3oT5ONe433n0vuUX1xOSUUlZRVWXMwmmgV42qX3KreojP5vLOVoYRlTR3bnuq5RDV6Ds1IgqmMKRCLiCFL2HGPs5xsY2CmSZ67rhMlkYvXeY9z8zirbPX6ersx6oM9Ztzg4lHecG2es5HB+CZ5uZkrKrfh7uvL1Kd57pKCUr1IP8r/NGaec3wQQ5ufBqzd25WhBKV+vP8jKPVX7RrWP8OO1m7piKSln0fZsUvYeI+1Y8Um7lFdzdzXj4WqmoKSCDpF+fHxPb9Jyipm+aDer9+XwyIB23NmnlS3YzN5wkO83ZTC0azOu7RKFm4uZLYfyeWHu9pP2rgrxcad7TCC9WgVzU8/oGmGrPj38xSa+Xl81sT7Mz4OFD1/RpHq/HJkCUR1TIBIRR3GqXpLkT9cz95cMXM0m3r/7Evq1rd1ZbnuOFHLjzBRyispwczHx0ajeZ12Sn3asmPnbMlm26yhr9h2jpNxK79hgpt3avcYw35Id2Tzy5SaOFpad9rNCfd3x93TDx8OVQG83+sdFMKxrc44UlnDru6vJLijFz9OVgpKa4enW3jE8fk0Hnv2+5gq+5oFedI0O4IctmbYhSxezCXcXM+WVViqsv/+683F34c4+rbjnsosIPkUwKq2oZPXeHBZuzyL/eDmj+l1EfItTT0IvKCknt6icmJCThwiX7TzCHf9ZY9tpPctSyt19W/GPoZ1O+/cidUeBqI4pEImII8u2lPDUt1sZ1i2Ka+KbndN7txzK582fdnJzrxj6x0Wc03tLyivJtpTSIsjLdpDviY4WlvLoV7+w6NdsQnzcuapDOFe1D6d9pC8tgrxPGko70b6jRdz67ioy8ktwNZu44eLmRAV6MWXhLgwDW6+W2QTXd2vOsl1HaoSv67pGMWFge9uWBKUVlWw9bGH9gVxmrT/EtoyqXcu93FwY0CmC67pGEd88gJ93HWXhr1ks3XGkxuR4swnuSGjF+AHtbL07B3OLeW/Ffj5fm05haQX3XX4Rjwxob9uHqrisggFvLONg7nHu6tOKqzuGc/u/12A2wXej+9WYmF9SXskHK/ezZl8OFVYDq2HgajYR5udBuJ8nXaMDSewYXiMMV1oN9h4pZEdWATszC/DxcOXmXtEnnTH4R0WlFfxyMJ/IAE9iQ31OeU+l1eDleb9itRqM69+u0e7crkBUxxSIRETOX25RGf5ebjUO462NzPwSftiSQf+4CNsE7YXbsxjz3w0UlVUS5ufB1Fu6k9A6hJLySr5ef5DU/bncltDyjKviDMNgwbYspizcxdbDltPeF+bnQWLHcApLK/l+U9XeU15uLvh4uGI2wbGiMiqtNX+Fdo0O5Klr49iTXci3mw6xYvcxmgd68eO4y/H1cGX0p+uZ80sGnaL8GZfYjvgWAWxMz+P5udtIzzl+xr+Pu/u2YuKQOMxmE7uyCvjbJ+vZlV1z7pavhyt39mnJnX1aEebrgclkotJqsG5/DvO3ZbFq7zG2Z1iwGlUh7+9D4vhL31Yn9TpOX7ybV37cAUDbcF9m3NaDNuG+1EZJeSVLdmRztLDMNgR6rLCMfUeLSMspJi7Kv8GOx1EgqmMKRCIijmN3dgELtmXz5x4tCPPzOPsbTsMwDDak5/HdxsPM3ZzBkYJSOjbzp3/HcK7uGEF88wBbz9fyXUeZ+O0W9v12Rl61fm1CGXVZLKXlVh79ahOWPwzvmUzwn7t6cVX7qtVlWZYSrn5tKYWnmEMV4e/BvZddRICXG64uJkrLrRwpKGX/sWLbHKShXaO4ukM4/zd7M8VllXi5udA+0o/2EX5sOphnm5QOVftNNQ/yIruglJyimkOXob7uth61m3tG89z1nW09W6kHcrnp7RQqrYZtyNLH3YWnr+vEsG7NT9qJvdqGtFw+W5PO/zZnnHaOWLUWQV5MuiGey9qGnfG+C6VAVMcUiEREmrZKq0FhaQUBXqef7FxRaWX/sSIqrAaVVgN/T7cau4Sn5xTzyJebWJ+WS3zzAPq0DmVgp8iT5h5tTM/jo5QDbDmUz67sAlzNZu69PJa/XXn6XpNvNhzikS831ZgH1ad1CNNGdrdtg2C1GszflsX0xbtPmvwe4OXG1R3D+VOHcHq2DCbC34P/rNjPC3O3YTWgc3N/RvWLpW+bUIZPX8mhvOMM6xbF34d05MFPN7D6t2Nqgn3cGXFxc27uFWPrMSqvtPLa/J3MXLrH9n3NA73o3NyfsgorpRVWArzcaBXqQ5ivB/9evs+2dcQlscG0DfclNtSH2FAfrmoffsrh1/OlQFTHFIhERKS2rFaj1r/Ui8sqMGHCy/3shwIv23mE+z9OpbiskgeubM3D/duddo+r42WVHMorJj3nON7uLvRoGXTKexfvyGbMpxtsPTou5qohtphgb+aO6YefpxsVlVbeXraXD1P2k2X5/bDkXq2CGN69BV+lprM+LQ+omruV1DuGXq2CT/t3UFRawSs/7uCDlP01DlX293Rl0z8G1OnWCApEdUyBSEREHMGhvOPkFZfRKarujl3JtpTw2dp0PluTxuHfJrF/9UCfkw5krqi0smTHET5bm87iHdk15k/5eboyeUSXc5rUvzu7gE3p+ew7WsS+o0W4u5p54+ZuddSqKgpEdUyBSEREmrpKq0HKnmP4e7nSpUXgGe/NspTwVepBZq0/SKhv1R5UZztk2B4UiOqYApGIiEjjcy6/v09/wI6IiIiIk1AgEhEREaenQCQiIiJOT4FIREREnJ4CkYiIiDg9BSIRERFxegpEIiIi4vQUiERERMTpKRCJiIiI03OqQDR9+nRatWqFp6cnvXv3Zs2aNfYuSURERByA0wSizz//nPHjx/OPf/yD9evX07VrVwYOHEh2dra9SxMRERE7c5pA9Prrr3Pvvfdy9913ExcXx8yZM/H29uY///mPvUsTERERO3OKQFRWVkZqaiqJiYm218xmM4mJiaSkpJx0f2lpKRaLpcZDREREmi6nCERHjx6lsrKSiIiIGq9HRESQmZl50v2TJk0iICDA9oiOjm6oUkVERMQOXO1dgCN64oknGD9+vO15fn4+MTEx6ikSERFpRKp/bxuGcdZ7nSIQhYaG4uLiQlZWVo3Xs7KyiIyMPOl+Dw8PPDw8bM+r/0LVUyQiItL4FBQUEBAQcMZ7nCIQubu706NHDxYuXMj1118PgNVqZeHChYwePfqs74+KiiI9PR0/Pz9MJlM9V9vwLBYL0dHRpKen4+/vb+9y6p0ztdeZ2gpqb1PmTG0FtbeuGIZBQUEBUVFRZ73XKQIRwPjx47nzzjvp2bMnl1xyCW+++SZFRUXcfffdZ32v2WymRYsWDVClffn7+zvFf3jVnKm9ztRWUHubMmdqK6i9deFsPUPVnCYQ3XzzzRw5coSnnnqKzMxMunXrxrx5806aaC0iIiLOx2kCEcDo0aNrNUQmIiIizsUplt3LmXl4ePCPf/yjxkTypsyZ2utMbQW1tylzpraC2msPJqM2a9FEREREmjD1EImIiIjTUyASERERp6dAJCIiIk5PgUhEREScngKRk5g0aRK9evXCz8+P8PBwrr/+enbs2FHjnpKSEpKTkwkJCcHX15cRI0acdNxJY/TSSy9hMpkYO3as7bWm1tZDhw5x2223ERISgpeXF/Hx8axbt8523TAMnnrqKZo1a4aXlxeJiYns2rXLjhWfv8rKSiZOnEhsbCxeXl60bt2a5557rsZZRY25vcuWLWPo0KFERUVhMpn45ptvalyvTdtycnJISkrC39+fwMBARo0aRWFhYQO2ovbO1N7y8nIee+wx4uPj8fHxISoqijvuuIPDhw/X+Iym0t4/uv/++zGZTLz55ps1Xm8s7a1NW7dv3851111HQEAAPj4+9OrVi7S0NNv1hvxZrUDkJJYuXUpycjKrVq1iwYIFlJeXM2DAAIqKimz3jBs3ju+//54vv/ySpUuXcvjwYW644QY7Vn3h1q5dy9tvv02XLl1qvN6U2pqbm0vfvn1xc3Pjhx9+YNu2bbz22msEBQXZ7pk8eTJTp05l5syZrF69Gh8fHwYOHEhJSYkdKz8/L7/8MjNmzOCtt95i+/btvPzyy0yePJlp06bZ7mnM7S0qKqJr165Mnz79lNdr07akpCS2bt3KggULmDNnDsuWLeO+++5rqCackzO1t7i4mPXr1zNx4kTWr1/PrFmz2LFjB9ddd12N+5pKe080e/ZsVq1adcojJxpLe8/W1j179tCvXz86dOjAkiVL+OWXX5g4cSKenp62exr0Z7UhTik7O9sAjKVLlxqGYRh5eXmGm5ub8eWXX9ru2b59uwEYKSkp9irzghQUFBht27Y1FixYYFxxxRXGQw89ZBhG02vrY489ZvTr1++0161WqxEZGWm88sorttfy8vIMDw8P47///W9DlFinhgwZYvzlL3+p8doNN9xgJCUlGYbRtNoLGLNnz7Y9r03btm3bZgDG2rVrbff88MMPhslkMg4dOtRgtZ+PP7b3VNasWWMAxoEDBwzDaJrtPXjwoNG8eXNjy5YtRsuWLY033njDdq2xtvdUbb355puN22677bTvaeif1eohclL5+fkABAcHA5Camkp5eTmJiYm2ezp06EBMTAwpKSl2qfFCJScnM2TIkBptgqbX1u+++46ePXty4403Eh4eTvfu3Xn33Xdt1/ft20dmZmaN9gYEBNC7d+9G2d4+ffqwcOFCdu7cCcCmTZtYvnw511xzDdD02nui2rQtJSWFwMBAevbsabsnMTERs9nM6tWrG7zmupafn4/JZCIwMBBoeu21Wq3cfvvtTJgwgU6dOp10vam012q1MnfuXNq1a8fAgQMJDw+nd+/eNYbVGvpntQKRE7JarYwdO5a+ffvSuXNnADIzM3F3d7f9kKkWERFBZmamHaq8MJ999hnr169n0qRJJ11ram3du3cvM2bMoG3btvz444888MADjBkzhg8++ADA1qY/ntvXWNv7+OOPc8stt9ChQwfc3Nzo3r07Y8eOJSkpCWh67T1RbdqWmZlJeHh4jeuurq4EBwc3+vaXlJTw2GOPMXLkSNsBoE2tvS+//DKurq6MGTPmlNebSnuzs7MpLCzkpZdeYtCgQcyfP5/hw4dzww03sHTpUqDhf1Y71VlmUiU5OZktW7awfPlye5dSL9LT03nooYdYsGBBjbHopspqtdKzZ09efPFFALp3786WLVuYOXMmd955p52rq3tffPEFn3zyCZ9++imdOnVi48aNjB07lqioqCbZXqlSXl7OTTfdhGEYzJgxw97l1IvU1FSmTJnC+vXrMZlM9i6nXlmtVgCGDRvGuHHjAOjWrRsrV65k5syZXHHFFQ1ek3qInMzo0aOZM2cOixcvpkWLFrbXIyMjKSsrIy8vr8b9WVlZREZGNnCVFyY1NZXs7GwuvvhiXF1dcXV1ZenSpUydOhVXV1ciIiKaTFsBmjVrRlxcXI3XOnbsaFupUd2mP67MaKztnTBhgq2XKD4+nttvv51x48bZegObWntPVJu2RUZGkp2dXeN6RUUFOTk5jbb91WHowIEDLFiwwNY7BE2rvT///DPZ2dnExMTYfnYdOHCAhx9+mFatWgFNp72hoaG4urqe9WdXQ/6sViByEoZhMHr0aGbPns2iRYuIjY2tcb1Hjx64ubmxcOFC22s7duwgLS2NhISEhi73glx99dVs3ryZjRs32h49e/YkKSnJ9uem0laAvn37nrSFws6dO2nZsiUAsbGxREZG1mivxWJh9erVjbK9xcXFmM01f3S5uLjY/h9nU2vviWrTtoSEBPLy8khNTbXds2jRIqxWK717927wmi9UdRjatWsXP/30EyEhITWuN6X23n777fzyyy81fnZFRUUxYcIEfvzxR6DptNfd3Z1evXqd8WdXg/9eqvNp2uKQHnjgASMgIMBYsmSJkZGRYXsUFxfb7rn//vuNmJgYY9GiRca6deuMhIQEIyEhwY5V150TV5kZRtNq65o1awxXV1fjhRdeMHbt2mV88sknhre3t/Hxxx/b7nnppZeMwMBA49tvvzV++eUXY9iwYUZsbKxx/PhxO1Z+fu68806jefPmxpw5c4x9+/YZs2bNMkJDQ41HH33Udk9jbm9BQYGxYcMGY8OGDQZgvP7668aGDRtsq6pq07ZBgwYZ3bt3N1avXm0sX77caNu2rTFy5Eh7NemMztTesrIy47rrrjNatGhhbNy4scbPrtLSUttnNJX2nsofV5kZRuNp79naOmvWLMPNzc145513jF27dhnTpk0zXFxcjJ9//tn2GQ35s1qByEkAp3y89957tnuOHz9u/O1vfzOCgoIMb29vY/jw4UZGRob9iq5DfwxETa2t33//vdG5c2fDw8PD6NChg/HOO+/UuG61Wo2JEycaERERhoeHh3H11VcbO3bssFO1F8ZisRgPPfSQERMTY3h6ehoXXXSR8fe//73GL8jG3N7Fixef8r/VO++80zCM2rXt2LFjxsiRIw1fX1/D39/fuPvuu42CggI7tObsztTeffv2nfZn1+LFi22f0VTaeyqnCkSNpb21aeu///1vo02bNoanp6fRtWtX45tvvqnxGQ35s9pkGCds7yoiIiLihDSHSERERJyeApGIiIg4PQUiERERcXoKRCIiIuL0FIhERETE6SkQiYiIiNNTIBIRERGnp0AkIiIiTk+BSEQcVqtWrXjzzTdrff+SJUswmUwnHQYpInI22qlaROrMlVdeSbdu3c4pxJzJkSNH8PHxwdvbu1b3l5WVkZOTQ0REBCaTqU5qOFdLlizhqquuIjc3l8DAQLvUICLnztXeBYiIczEMg8rKSlxdz/7jJyws7Jw+293dncjIyPMtTUScmIbMRKRO3HXXXSxdupQpU6ZgMpkwmUzs37/fNoz1ww8/0KNHDzw8PFi+fDl79uxh2LBhRERE4OvrS69evfjpp59qfOYfh8xMJhP/+te/GD58ON7e3rRt25bvvvvOdv2PQ2bvv/8+gYGB/Pjjj3Ts2BFfX18GDRpERkaG7T0VFRWMGTOGwMBAQkJCeOyxx7jzzju5/vrrT9vWAwcOMHToUIKCgvDx8aFTp07873//Y//+/Vx11VUABAUFYTKZuOuuuwCwWq1MmjSJ2NhYvLy86Nq1K1999dVJtc+dO5cuXbrg6enJpZdeypYtW876vSJy4RSIRKROTJkyhYSEBO69914yMjLIyMggOjradv3xxx/npZdeYvv27XTp0oXCwkIGDx7MwoUL2bBhA4MGDWLo0KGkpaWd8XueeeYZbrrpJn755RcGDx5MUlISOTk5p72/uLiYV199lY8++ohly5aRlpbGI488Yrv+8ssv88knn/Dee++xYsUKLBYL33zzzRlrSE5OprS0lGXLlrF582ZefvllfH19iY6O5uuvvwZgx44dZGRkMGXKFAAmTZrEhx9+yMyZM9m6dSvjxo3jtttuY+nSpTU+e8KECbz22musXbuWsLAwhg4dSnl5+Rm/V0TqgCEiUkeuuOIK46GHHqrx2uLFiw3A+Oabb876/k6dOhnTpk2zPW/ZsqXxxhtv2J4DxpNPPml7XlhYaADGDz/8UOO7cnNzDcMwjPfee88AjN27d9veM336dCMiIsL2PCIiwnjllVdszysqKoyYmBhj2LBhp60zPj7eePrpp0957Y81GIZhlJSUGN7e3sbKlStr3Dtq1Chj5MiRNd732Wef2a4fO3bM8PLyMj7//POzfq+IXBjNIRKRBtGzZ88azwsLC3n66aeZO3cuGRkZVFRUcPz48bP2EHXp0sX2Zx8fH/z9/cnOzj7t/d7e3rRu3dr2vFmzZrb78/PzycrK4pJLLrFdd3FxoUePHlit1tN+5pgxY3jggQeYP38+iYmJjBgxokZdf7R7926Ki4vp379/jdfLysro3r17jdcSEhJsfw4ODqZ9+/Zs3779vL5XRGpPQ2Yi0iB8fHxqPH/kkUeYPXs2L774Ij///DMbN24kPj6esrKyM36Om5tbjecmk+mM4eVU9xsXuLj2nnvuYe/evdx+++1s3ryZnj17Mm3atNPeX1hYCMDcuXPZuHGj7bFt27Ya84jq+ntFpPYUiESkzri7u1NZWVmre1esWMFdd93F8OHDiY+PJzIykv3799dvgX8QEBBAREQEa9eutb1WWVnJ+vXrz/re6Oho7r//fmbNmsXDDz/Mu+++C1T9HVR/TrW4uDg8PDxIS0ujTZs2NR4nzrMCWLVqle3Pubm57Ny5k44dO571e0XkwmjITETqTKtWrVi9ejX79+/H19eX4ODg097btm1bZs2axdChQzGZTEycOPGMPT315cEHH2TSpEm0adOGDh06MG3aNHJzc8+4j9HYsWO55ppraNeuHbm5uSxevNgWWlq2bInJZGLOnDkMHjwYLy8v/Pz8eOSRRxg3bhxWq5V+/fqRn5/PihUr8Pf3584777R99rPPPktISAgRERH8/e9/JzQ01Lbi7UzfKyIXRj1EIlJnHnnkEVxcXIiLiyMsLOyM84Fef/11goKC6NOnD0OHDmXgwIFcfPHFDVhtlccee4yRI0dyxx13kJCQgK+vLwMHDsTT0/O076msrCQ5OZmOHTsyaNAg2rVrxz//+U8AmjdvzjPPPMPjjz9OREQEo0ePBuC5555j4sSJTJo0yfa+uXPnEhsbW+OzX3rpJR566CF69OhBZmYm33//fY1ep9N9r4hcGO1ULSJyAqvVSseOHbnpppt47rnnGux7tcO1iH1pyExEnNqBAweYP38+V1xxBaWlpbz11lvs27ePW2+91d6liUgD0pCZiDg1s9nM+++/T69evejbty+bN2/mp59+0twcESejITMRERFxeuohEhEREaenQCQiIiJOT4FIREREnJ4CkYiIiDg9BSIRERFxegpEIiIi4vQUiERERMTpKRCJiIiI0/t/BXv1LtGfI88AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.optim import SGD, Adam\n",
    "from copy import deepcopy\n",
    "\n",
    "# 将数据适配为PyTorch数据集\n",
    "class SeqLabelDataset(Dataset):\n",
    "    def __init__(self, tokens, labels):\n",
    "        self.tokens = deepcopy(tokens)\n",
    "        self.labels = deepcopy(labels)\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        return self.tokens[idx], self.labels[idx]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "    \n",
    "    # 将每个批次转化为PyTorch张量\n",
    "    @classmethod\n",
    "    def collate_batch(cls, inputs):\n",
    "        input_ids, labels, masks = [], [], []\n",
    "        max_len = -1\n",
    "        for ids, tags in inputs:\n",
    "            input_ids.append(ids)\n",
    "            labels.append(tags)\n",
    "            masks.append([1] * len(tags))\n",
    "            max_len = max(max_len, len(ids))\n",
    "        for ids, tags, msks in zip(input_ids, labels, masks):\n",
    "            pad_len = max_len - len(ids)\n",
    "            ids.extend([0] * pad_len)\n",
    "            tags.extend([0] * pad_len)\n",
    "            msks.extend([0] * pad_len)\n",
    "        input_ids = torch.tensor(np.array(input_ids),\\\n",
    "            dtype=torch.long)\n",
    "        labels = torch.tensor(np.array(labels), dtype=torch.long)\n",
    "        masks = torch.tensor(np.array(masks), dtype=torch.uint8)\n",
    "        return {'input_ids': input_ids, 'masks': masks,\\\n",
    "                'labels': labels}\n",
    "    \n",
    "# 准备数据\n",
    "train_dataset = SeqLabelDataset(train_X, train_Y)\n",
    "test_dataset = SeqLabelDataset(test_X, test_Y)\n",
    "\n",
    "from tqdm import tqdm, trange\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def train(model, batch_size, epochs, learning_rate):\n",
    "    train_dataloader = DataLoader(train_dataset,\n",
    "        batch_size=batch_size, shuffle=True, \n",
    "        collate_fn=SeqLabelDataset.collate_batch)\n",
    "    optimizer = Adam(model.parameters(), lr=learning_rate)\n",
    "    model.zero_grad()\n",
    "    tr_step = []\n",
    "    tr_loss = []\n",
    "    global_step = 0\n",
    "    \n",
    "    with trange(epochs, desc='epoch', ncols=60) as pbar:\n",
    "        for epoch in pbar:\n",
    "            model.train()\n",
    "            for step, batch in enumerate(train_dataloader):\n",
    "                global_step += 1\n",
    "                loss = model(**batch)\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "                model.zero_grad()\n",
    "                pbar.set_description(f'epoch-{epoch}, '+\\\n",
    "                    f'loss={loss.item():.2f}')\n",
    "                \n",
    "                if epoch > 0:\n",
    "                    tr_step.append(global_step)\n",
    "                    tr_loss.append(loss.item())\n",
    "\n",
    "    # 打印损失曲线\n",
    "    plt.plot(tr_step, tr_loss, label='train loss')\n",
    "    plt.xlabel('training steps')\n",
    "    plt.ylabel('loss')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "                \n",
    "vocab_size = len(dataset.token2id)\n",
    "hidden_size = 32\n",
    "n_tags = len(dataset.label2id)\n",
    "crf = CRF(vocab_size, hidden_size, n_tags)\n",
    "batch_size = 128\n",
    "epochs = 20\n",
    "learning_rate = 1e-2\n",
    "train(crf, batch_size, epochs, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4a1e7e25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.5408805031446541, recall = 0.24606580829756797, f1 = 0.33824975417895775\n",
      "precision = 0.48588410104011887, recall = 0.20024494794856093, f1 = 0.2836079791847355\n"
     ]
    }
   ],
   "source": [
    "# 验证效果\n",
    "\n",
    "def evaluate(X, Y, model, batch_size):\n",
    "    dataset = SeqLabelDataset(X, Y)\n",
    "    dataloader = DataLoader(dataset, batch_size=batch_size,\\\n",
    "        collate_fn=SeqLabelDataset.collate_batch)\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        P = []\n",
    "        for batch in dataloader:\n",
    "            preds = model.decode(batch['input_ids'], batch['masks'])\n",
    "            P.extend(preds)\n",
    "\n",
    "    p, r, f = compute_metric(Y, P)\n",
    "    print(f'precision = {p}, recall = {r}, f1 = {f}')\n",
    "    \n",
    "evaluate(train_X, train_Y, crf, batch_size)\n",
    "evaluate(test_X, test_Y, crf, batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc369a17",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "这里介绍基于双向长短期记忆-条件随机场（BiLSTM-CRF）结构的代码实现。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "990c5bc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "# 定义模型，将长短期记忆的输出输入给条件随机场，\n",
    "# 利用条件随机场计算损失和解码\n",
    "class LSTM_CRF(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, num_layers,\\\n",
    "                 dropout, n_tags):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "        self.lstm = nn.LSTM(input_size=hidden_size,\n",
    "                            hidden_size=hidden_size,\n",
    "                            num_layers=num_layers,\n",
    "                            batch_first=False,\n",
    "                            dropout=dropout,\n",
    "                            bidirectional=True)\n",
    "        self.crf = CRFLayer(n_tags, hidden_size * 2)\n",
    "    \n",
    "    def forward(self, input_ids, masks, labels):\n",
    "        \"\"\"\n",
    "        input_ids/masks/labels: batch_size * seq_len\n",
    "        \"\"\"\n",
    "        # 将输入序列转化为词嵌入序列\n",
    "        # batch_size * seq_len * embed_size\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        labels = torch.transpose(labels, 0, 1)\n",
    "        # 输入长短期记忆得到隐状态\n",
    "        hidden_states, _ = self.lstm(embed)\n",
    "        # 隐状态和标签、掩码输入条件随机场计算损失\n",
    "        llh = self.crf(hidden_states, labels, masks)\n",
    "        return -llh\n",
    "    \n",
    "    def decode(self, input_ids, masks):\n",
    "        # 输入长短期记忆得到隐状态\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        hidden_states, _ = self.lstm(embed)\n",
    "        # 调用条件随机场进行解码\n",
    "        return self.crf.decode(hidden_states, masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ab41a9f0",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-19, loss=36.92: 100%|█| 20/20 [04:21<00:00, 13.10s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABuLUlEQVR4nO3dd1hUV/4/8PedCgPM0JvSFJQilqhR1FgSY40xxjRjNGZTfmYxRWM2m90k32yyq2mbusbUTdnEdDWxd7Gjoig2FEVAqYL0NuX+/piZC6OAqAMD4/v1PDwLc+/cOcds8J1zPuccQRRFEUREREROSuboBhARERG1JYYdIiIicmoMO0REROTUGHaIiIjIqTHsEBERkVNj2CEiIiKnxrBDRERETk3h6AZ0BCaTCbm5ufDw8IAgCI5uDhEREbWCKIqoqKhAcHAwZLLmx28YdgDk5uYiJCTE0c0gIiKia5CTk4OuXbs2e51hB4CHhwcA8x+WVqt1cGuIiIioNcrLyxESEiL9Pd4chh1AmrrSarUMO0RERJ3MlUpQWKBMRERETo1hh4iIiJwaww4RERE5NdbsEBGR0zMajdDr9Y5uBl0lpVIJuVx+3c9h2CEiIqcliiLy8/NRWlrq6KbQNfL09ERgYOB17YPHsENERE7LGnT8/f2h0Wi4cWwnIooiqqurUVhYCAAICgq65mcx7BARkVMyGo1S0PHx8XF0c+gauLq6AgAKCwvh7+9/zVNaLFAmIiKnZK3R0Wg0Dm4JXQ/rP7/rqbli2CEiIqfGqavOzR7//Bh2iIiIyKkx7BAREZFTY9ghIiJycuHh4Xj//fcd/gxH4WosBxBFEXUGE1yU179REhEROZ+RI0eib9++dgsX+/btg5ubm12e1RlxZMcB/vz9Adz8r40oqap3dFOIiKiTEkURBoOhVff6+fnd0KvSGHYcYN/ZEpTXGpCeX+HophAR3VBEUUR1vcEhX6IotqqNs2bNQlJSEj744AMIggBBEHD27Fls3boVgiBgzZo16N+/P9RqNXbs2IHTp09j8uTJCAgIgLu7OwYOHIiNGzfaPPPSKShBEPDFF19gypQp0Gg0iIqKwh9//HFVf5bZ2dmYPHky3N3dodVqcd9996GgoEC6fujQIYwaNQoeHh7QarXo378/9u/fDwDIysrCpEmT4OXlBTc3N8TFxWH16tVX9flXg9NYDlBea7D8L89pISJqTzV6I2JfWeeQzz722lhoVFf+a/eDDz7AyZMn0atXL7z22msAzCMzZ8+eBQD89a9/xTvvvINu3brBy8sLOTk5mDBhAv71r39BrVbj22+/xaRJk5Ceno7Q0NBmP+cf//gH3nrrLbz99tv46KOPMH36dGRlZcHb2/uKbTSZTFLQSUpKgsFgQGJiIu6//35s3boVADB9+nT069cPixcvhlwuR2pqKpRKJQAgMTER9fX12LZtG9zc3HDs2DG4u7tf8XOvFcNOO6szGFFvMAEAymsYdoiIyJZOp4NKpYJGo0FgYOBl11977TXcfvvt0s/e3t7o06eP9PPrr7+OZcuW4Y8//sCcOXOa/ZxZs2Zh2rRpAIAFCxbgww8/xN69ezFu3LgrtnHTpk1IS0tDZmYmQkJCAADffvst4uLisG/fPgwcOBDZ2dl4/vnnER0dDQCIioqS3p+dnY2pU6ciPj4eANCtW7crfub1YNhpZ5W1DfOr5bWtm2slIiL7cFXKcey1sQ77bHsYMGCAzc+VlZV49dVXsWrVKuTl5cFgMKCmpgbZ2dktPqd3797S925ubtBqtdI5VFdy/PhxhISESEEHAGJjY+Hp6Ynjx49j4MCBmDdvHh577DH873//w+jRo3Hvvfeie/fuAICnn34aTz75JNavX4/Ro0dj6tSpNu2xN9bstLOKRgGngtNYRETtShAEaFQKh3zZayfnS1dVzZ8/H8uWLcOCBQuwfft2pKamIj4+HvX1LS+CsU4pNf6zMZlMdmkjALz66qs4evQoJk6ciM2bNyM2NhbLli0DADz22GM4c+YMZsyYgbS0NAwYMAAfffSR3T77Ugw77axx2Cmv4cgOERFdTqVSwWg0turenTt3YtasWZgyZQri4+MRGBgo1fe0lZiYGOTk5CAnJ0d67dixYygtLUVsbKz0Wo8ePTB37lysX78ed999N7766ivpWkhICGbPno2lS5fiueeew+eff95m7WXYaWeNR3NYoExERE0JDw9HcnIyzp49iwsXLrQ44hIVFYWlS5ciNTUVhw4dwoMPPmjXEZqmjB49GvHx8Zg+fToOHDiAvXv3YubMmRgxYgQGDBiAmpoazJkzB1u3bkVWVhZ27tyJffv2ISYmBgDw7LPPYt26dcjMzMSBAwewZcsW6VpbYNhpZxV1jUd2GHaIiOhy8+fPh1wuR2xsLPz8/Fqsv3n33Xfh5eWFIUOGYNKkSRg7dixuuummNm2fIAj4/fff4eXlheHDh2P06NHo1q0bfvrpJwCAXC5HcXExZs6ciR49euC+++7D+PHj8Y9//AMAYDQakZiYiJiYGIwbNw49evTAxx9/3HbtFVu78N+JlZeXQ6fToaysDFqttk0/69eUc5j/yyEAwOBu3vjxiYQ2/TwiohtVbW0tMjMzERERARcXF0c3h65RS/8cW/v3N0d22pnNNBZrdoiIiNocw047sylQZs0OERFRm2PYaWeVrNkhIiJqVww77azxNFZFnQEm0w1fMkVE1KZYmtq52eOfH8NOO2u8a7IoApX1rNshImoL1k3zqqurHdwSuh7Wf36XboJ4NXhcRDurvOSIiPIaPbQu1/4PkIiImiaXy+Hp6SkdgaDRaOy2izG1PVEUUV1djcLCQnh6ekIuv/bjNhh22tmlR0SU1xgALwc1hojIyVkP0mztmU/U8Xh6ejZ5IOrVcGjYWbx4MRYvXixtax0XF4dXXnkF48ePB2BeW//cc8/hxx9/RF1dHcaOHYuPP/4YAQEB0jOys7Px5JNPYsuWLXB3d8fDDz+MhQsXQqHomDmu4tKRHa7IIiJqM4IgICgoCP7+/tDr+fu2s1Eqldc1omPl0ETQtWtXvPHGG4iKioIoivjmm28wefJkHDx4EHFxcZg7dy5WrVqFX375BTqdDnPmzMHdd9+NnTt3AjDvwDhx4kQEBgZi165dyMvLw8yZM6FUKrFgwQJHdq1Z1rCjlAvQG8XLwg8REdmfXC63y1+a1Dl1uB2Uvb298fbbb+Oee+6Bn58flixZgnvuuQcAcOLECcTExGD37t0YPHgw1qxZgzvuuAO5ubnSaM8nn3yCF154AUVFRVCpVK36zPbcQbnX/61DZZ0Bod4aZJdU49/39sHU/l3b9DOJiIicUafbQdloNOLHH39EVVUVEhISkJKSAr1ej9GjR0v3REdHIzQ0FLt37wYA7N69G/Hx8TbTWmPHjkV5eTmOHj3a7GfV1dWhvLzc5qs9GE2itM9OF09XAJzGIiIiamsODztpaWlwd3eHWq3G7NmzsWzZMsTGxiI/Px8qlQqenp429wcEBCA/Px8AkJ+fbxN0rNet15qzcOFC6HQ66SskJMS+nWpG4w0Fu3hZwg6PjCAiImpTDg87PXv2RGpqKpKTk/Hkk0/i4YcfxrFjx9r0M1988UWUlZVJXzk5OW36eVbWlVgquQy+7moAHNkhIiJqaw5fsqRSqRAZGQkA6N+/P/bt24cPPvgA999/P+rr61FaWmozulNQUCAtQQsMDMTevXttnldQUCBda45arYZarbZzT67MOrLj4aKAh4v5j55HRhAREbUth4/sXMpkMqGurg79+/eHUqnEpk2bpGvp6enIzs5GQkICACAhIQFpaWk2+yds2LABWq0WsbGx7d72K7GuvPJwUUDrat5IkCM7REREbcuhIzsvvvgixo8fj9DQUFRUVGDJkiXYunUr1q1bB51Oh0cffRTz5s2Dt7c3tFotnnrqKSQkJGDw4MEAgDFjxiA2NhYzZszAW2+9hfz8fLz00ktITEx0yMjNlVinsTxclNBKIzus2SEiImpLDg07hYWFmDlzJvLy8qDT6dC7d2+sW7cOt99+OwDgvffeg0wmw9SpU202FbSSy+VYuXIlnnzySSQkJMDNzQ0PP/wwXnvtNUd1qUXWkR13NUd2iIiI2otDw86XX37Z4nUXFxcsWrQIixYtavaesLAwrF692t5NaxM201guDDtERETtocPV7DizhrCjhM6V01hERETtgWGnHTXU7DSM7FTU6mEydahNrImIiJwKw047arz03FqzYxKBqnqO7hAREbUVhp121LhmR62QQSWX2bxORERE9sew044aLz0XBAFaa90Oi5SJiIjaDMNOOypvNLIDoGFFFouUiYiI2gzDTjuqbLQaCwA8rHvt8MgIIiKiNsOw044q6syhxl1tHdnhNBYREVFbY9hpR9ZCZO1l01gMO0RERG2FYaediKJos6kggEYFyqzZISIiaisMO+2kVm+C0bJ54OUFyhzZISIiaisMO+3EuuxcJgAalRwAeBgoERFRO2DYaSfljU48FwQBQKMCZS49JyIiajMMO+2k8YaCVhzZISIiansMO+2k8blYVlLNjiXs/GfzKSQuOSDV9hAREdH1Y9hpJxW1TYQd14ZprLRzZXhn/UmsOpyH43nlDmkjERGRM2LYaSdNTmM1Gtn594Z06fWqOtbwEBER2QvDTjtpemTHHHZKq/XYml4kvV5db2zfxhERETkxhp120lTYafx9Yww7RERE9sOw004qpKXnDdNYrko5FDLzMnSVXIboQA8AQHU9p7GIiIjshWGnnTTU7DSM5giCIE1lTR8cim5+bgA4skNERGRPDDvtxLr0XHvJ1NWUfl0QF6xF4qhIuCrN1xh2iIiI7KfpohGyu0sPAbV6+Y5Y6Xs3tfkYiRpOYxEREdkNR3baiXUay13dfL50tZyZVcWRHSIiIrth2GkHJpOIwoo6AM2vwAIADaexiIiI7I5hpx0knSxCXlkt3NUKxAZrm72P01hERET2x7DTDj7ddhoA8OCg0MtqdhrjNBYREZH9Mey0scPnSrHnTAkUMgGzhoS3eK+byjyNVcOwQ0REZDcMO23s8+2ZAIA7+wQj2NO1xXsbRnY4jUVERGQvDDttKKekGqvT8gAAj93S7Yr3a1TWmh2O7BAREdkLw04b+mrnWRhNIm6J8m2xMNlKo+JqLCIiIntj2GkjVXUG/LQvGwDweCtGdYCGkR2ejUVERGQ/3EG5jbipFfhl9hCsOJyLW6J8W/WehrDDkR0iIiJ7YdhpQ7HB2lZNX1lZp7Fq9EaYTCJklhPRiYiI6NpxGqsDsY7siCJQa+DoDhERkT0w7HQgrkq59D2nsoiIiOyDYacDkckEKfBw+TkREZF9MOx0MBpuLEhERGRXDDsdjEbNFVlERET2xLDTwWiUlo0F6xh2iIiI7IFhp4Nx5caCREREdsWw08G4WaaxavQc2SEiIrIHhp0OxtUyjVXFaSwiIiK7YNjpYNzUnMYiIiKyJ4adDsa69Jz77BAREdkHw04HI01jMewQERHZBcNOByMVKHMai4iIyC4YdjqYhqXnHNkhIiKyB4eGnYULF2LgwIHw8PCAv78/7rrrLqSnp9vcM3LkSAiCYPM1e/Zsm3uys7MxceJEaDQa+Pv74/nnn4fB0DlHRtxUlk0FGXaIiIjsQuHID09KSkJiYiIGDhwIg8GAv/3tbxgzZgyOHTsGNzc36b7HH38cr732mvSzRqORvjcajZg4cSICAwOxa9cu5OXlYebMmVAqlViwYEG79sceuKkgERGRfTk07Kxdu9bm56+//hr+/v5ISUnB8OHDpdc1Gg0CAwObfMb69etx7NgxbNy4EQEBAejbty9ef/11vPDCC3j11VehUqkue09dXR3q6uqkn8vLy+3Uo+vXcBAoR3aIiIjsoUPV7JSVlQEAvL29bV7//vvv4evri169euHFF19EdXW1dG337t2Ij49HQECA9NrYsWNRXl6Oo0ePNvk5CxcuhE6nk75CQkLaoDfXxjqNxaXnRERE9uHQkZ3GTCYTnn32WQwdOhS9evWSXn/wwQcRFhaG4OBgHD58GC+88ALS09OxdOlSAEB+fr5N0AEg/Zyfn9/kZ7344ouYN2+e9HN5eXmHCTycxiIiIrKvDhN2EhMTceTIEezYscPm9SeeeEL6Pj4+HkFBQbjttttw+vRpdO/e/Zo+S61WQ61WX1d724qGq7GIiIjsqkNMY82ZMwcrV67Eli1b0LVr1xbvHTRoEAAgIyMDABAYGIiCggKbe6w/N1fn05FpuBqLiIjIrhwadkRRxJw5c7Bs2TJs3rwZERERV3xPamoqACAoKAgAkJCQgLS0NBQWFkr3bNiwAVqtFrGxsW3S7rbE4yKIiIjsy6HTWImJiViyZAl+//13eHh4SDU2Op0Orq6uOH36NJYsWYIJEybAx8cHhw8fxty5czF8+HD07t0bADBmzBjExsZixowZeOutt5Cfn4+XXnoJiYmJHXaqqiXWAuV6owl6owlKeYcYfCMiIuq0HPo36eLFi1FWVoaRI0ciKChI+vrpp58AACqVChs3bsSYMWMQHR2N5557DlOnTsWKFSukZ8jlcqxcuRJyuRwJCQl46KGHMHPmTJt9eToTa4EywKksIiIie3DoyI4oii1eDwkJQVJS0hWfExYWhtWrV9urWQ6lUsigkAkwmERU1xugc1U6uklERESdGudIOiCuyCIiIrIfhp0OSMONBYmIiOyGYacDko6MqOPGgkRERNeLYacD0qgt01h6juwQERFdL4adDkij5DQWERGRvTDsdEDWkR1OYxEREV0/hp0OSNpFmdNYRERE141hpwNyVfJ8LCIiInth2OmA3KwFypzGIiIium4MOx2QKzcVJCIishuGnQ7IehhoFcMOERHRdWPY6YCkAuV6TmMRERFdL4adDojTWERERPbDsNMBWaexGHaIiIiuH8NOB9QwssNpLCIiouvFsNMBaTiNRUREZDcMOx2QhtNYREREdsOw0wFxZIeIiMh+GHY6IGuBMpeeExERXT+GnQ5IKlDWGyGKooNbQ0RE1Lkx7HRA1mksUQRq9SYHt4aIiKhzY9jpgFyVcun7Kk5lERERXReGnQ5IJhOkwFPDImUiIqLrwrDTQfl5qAEA/92Z6eCWEBERdW4MOx3UC+OiAQBf7TyLb3efdWxjiIiIOjGGnQ5qYu8g/GVcTwDAq38cxZYThQ5uERERUefEsNOBPTmiO+4fEAKTCDz1w0EUV9Y5uklERESdDsNOByYIAv45pRd6Bnigss6ADccKHN0kIiKiTodhp4NTymWY1CcIALDuaL6DW0NERNT5MOx0AmPjAgEAOzOKUVGrd3BriIiIOheGnU4g0t8d3XzdUG80IelkkaObQ0RE1Kkw7HQCgiDg9rgAAMC6o6zbISIiuhoMO52EdSpry4lC1Bm4qzIREVFrMex0En27esLfQ43KOgN2ny52dHOIiIg6DYadTkImE3B7LKeyiIiIrhbDTidincracKwARpPo4NYQERF1Dgw7ncjgbj5wVcpxobIOWcVVjm4OERFRp8Cw04moFDLpNPSSqnoHt4aIiKhzYNjpZLzcVAAYdoiIiFqLYaeT8dYoAQCl1dxJmYiIqDUYdjoZL41lZKeaIztEREStwbDTyVinsS5yGouIiKhVGHY6GW9r2OHIDhERUasw7HQynpaanZIq1uwQERG1BsNOJ+Ntqdkp5cgOERFRqzDsdDLS0nOGHSIiolZh2OlkrKuxWKBMRETUOg4NOwsXLsTAgQPh4eEBf39/3HXXXUhPT7e5p7a2FomJifDx8YG7uzumTp2KggLbgzCzs7MxceJEaDQa+Pv74/nnn4fBYGjPrrQbLzdzzU5ZjZ7nYxEREbWCQ8NOUlISEhMTsWfPHmzYsAF6vR5jxoxBVVXDuU9z587FihUr8MsvvyApKQm5ubm4++67petGoxETJ05EfX09du3ahW+++QZff/01XnnlFUd0qc1ZR3ZMIlBewyJlIiKiKxFEUewwwwNFRUXw9/dHUlIShg8fjrKyMvj5+WHJkiW45557AAAnTpxATEwMdu/ejcGDB2PNmjW44447kJubi4CAAADAJ598ghdeeAFFRUVQqVRX/Nzy8nLodDqUlZVBq9W2aR/tIf7/1qGizoDNz41ANz93RzeHiIjIIVr793eHqtkpKysDAHh7ewMAUlJSoNfrMXr0aOme6OhohIaGYvfu3QCA3bt3Iz4+Xgo6ADB27FiUl5fj6NGjTX5OXV0dysvLbb46Ey/utUNERNRqHSbsmEwmPPvssxg6dCh69eoFAMjPz4dKpYKnp6fNvQEBAcjPz5fuaRx0rNet15qycOFC6HQ66SskJMTOvWlbDYeBchqLiIjoSjpM2ElMTMSRI0fw448/tvlnvfjiiygrK5O+cnJy2vwz7cnLsrFga0Z2Xlx6GI9/ux+1emNbN4uIiKhD6hBhZ86cOVi5ciW2bNmCrl27Sq8HBgaivr4epaWlNvcXFBQgMDBQuufS1VnWn633XEqtVkOr1dp8dSberVx+nlVchR/25mDDsQJ8vCWjPZpGRETU4Tg07IiiiDlz5mDZsmXYvHkzIiIibK73798fSqUSmzZtkl5LT09HdnY2EhISAAAJCQlIS0tDYWGhdM+GDRug1WoRGxvbPh1pZ63dWHBHxgXp+8VJp3GyoKJN20VERNQROTTsJCYm4rvvvsOSJUvg4eGB/Px85Ofno6amBgCg0+nw6KOPYt68ediyZQtSUlLwyCOPICEhAYMHDwYAjBkzBrGxsZgxYwYOHTqEdevW4aWXXkJiYiLUarUju9dmrNNYpVeo2dlxyhx2XJQy6I0iXlyaBhP35iEiohuMQ8PO4sWLUVZWhpEjRyIoKEj6+umnn6R73nvvPdxxxx2YOnUqhg8fjsDAQCxdulS6LpfLsXLlSsjlciQkJOChhx7CzJkz8dprrzmiS+2iNSM7RpOIXaeLAQD/vrcv3FRypGRdxJK92e3SRiIioo5C4cgPb80WPy4uLli0aBEWLVrU7D1hYWFYvXq1PZvWobXmMNC082Uoq9HDw0WBsXEBKKroiVdXHMOba0/gnv5d4aKUt1dziYiIHKpDFCjT1fHUWJeeNx92dpwqAgAM6e4DhVyGGQnh0LkqUVFrQOaFqmbfR0RE5GwYdjohb2lTweZrdrZb6nWGRfkBAOQyAd383ACAYYeIiG4oDDudkFSgXF0vFRw//8sh3PfJbhRW1KKqzoAD2RcBALdE+krvi/Bh2CEiohuPQ2t26Np4Nj4MtNZ8+vkvKecAADO+2Iv/N6Ib9EYRXTxdEeajkd4X4cuwQ0RENx6GnU5IpZDBQ61ARZ0BJVX1yCqplq6lF1Tg+V8PAwBuifKFIAjStQhOYxER0Q2I01idlKeb9cgIPQ7llAIAbg73hrebCkbL1NawKF+b91ztyE6dwSg9i4iIqLNi2OmkGh8ZYQ07E3sH4ds/3QytiwIeagWGRdqGnXBLzU5JVX2Ly9YBoFZvxPj3t2Pih9u5ESEREXVqnMbqpBpvLHjoXBkAoE+IJ3p10WHz/JGoN5ik2h4rN7UCAVo1CsrrkHmhCv1CVZc91yol6yLOWEaAiirrEKB1aaOeEBERtS2O7HRSXpYgk3auDCVV9VDKBcQEeQAAfN3VCPZ0bfJ91qmss8UtT2XtbHSu1rmLNfZoMhERkUMw7HRS1rCzJd18AGpMkBZqxZV3RY7wdQcAZBZdIexYjpoAgPOlDDtERNR5Mex0Ut6WAmXrqEufrp6tel83y8jOmUZFyifyy3Eiv1z6uaxGj7RzpdLP5zmyQ0REnRjDTid1aT1OnxDPVr3v0hVZF6vqcffHu3DXop3IL6sFACSfKUbjmuTzpdWXPYeIiKizYNjppKxHRlj1DdG16n3hjcKOKIpYlZaH6nojavUmfLP7LABIp6V7qM316xzZISKizoxhp5PyajSy465WoJulFudKQr01kAlAdb0RRRV1+CM1V7r2/Z4sVNUZsMNSnDypbzAA1uwQEVHnxrDTSXlZanYAIL6LDjKZ0MLdDVQKGUK8zUdIbD91AXvPlkAQgCCdC8prDVi0JQMZhZUQBGDqTV0BmEd2RJF77RARUed0TWHnm2++wapVq6Sf//KXv8DT0xNDhgxBVlaW3RpHzfNuNLLT2nodK2vdzn+2ZAAABkf44M8juwMAFiedBgDEBWsRF6wFAFTVG1FW0/wJ60RERB3ZNYWdBQsWwNXVvI/L7t27sWjRIrz11lvw9fXF3Llz7dpAalrjAuXW1utYXVqkPLlvMO7pHwIvjRLWAZyh3X3hopTD1938Odxrh4iIOqtrCjs5OTmIjIwEACxfvhxTp07FE088gYULF2L79u12bSA1TaWQoYunK1RyGW4K87qq91rDDgCo5DKMjw+Cq0qOhwaHSa8PsRw10cWyOSHrdoiIqLO6prDj7u6O4mLzip3169fj9ttvBwC4uLigpoZ/KbaXHx4fjKV/HgJ/j6s7yqFx2BkV7Qedq7n+Z2ZCONzVCnhqlBgYbg5QXbwsYYcjO0RE1Eld09lYt99+Ox577DH069cPJ0+exIQJEwAAR48eRXh4uD3bRy0I9dFc0/sah527+naRvvfzUGP107dAEACNyvx/DY7sEBFRZ3dNIzuLFi1CQkICioqK8Ntvv8HHxwcAkJKSgmnTptm1gWR/wTpXxAVrEenvjlHR/jbXQn000motoFHY4cgOERF1UoLINcUoLy+HTqdDWVkZtFqto5vTLkRRhNEkQiFvOe9uOFaAx7/dj/guOqx4alg7tY6IiOjKWvv39zWN7KxduxY7duyQfl60aBH69u2LBx98EBcvXryWR1I7EwThikEH4DQWERF1ftcUdp5//nmUl5sPjkxLS8Nzzz2HCRMmIDMzE/PmzbNrA8mxrAXKJVX1qK43OLg1REREV++aCpQzMzMRGxsLAPjtt99wxx13YMGCBThw4IBUrEzOQeeqhIdagYo6A3JLaxDp79HkfeW1etz3yW54u6nw31kD4aKUt3NLiYiImnZNIzsqlQrV1eaTsDdu3IgxY8YAALy9vaURH3Ie1tGdljYW/M/mDJzIr8Cu08X4v9+PtlfTiIiIruiaws6wYcMwb948vP7669i7dy8mTpwIADh58iS6du1q1waS411at7P+aD62nyqSrmcVV+GrnZnSzz/tz8GPe7Pbt5FERETNuKaw85///AcKhQK//vorFi9ejC5dzHu1rFmzBuPGjbNrA8nxGm8suOJQLp74XwpmfLkXi7eaz9FauPoE9EYRt0T54vmxPQEAr/x+FIdySh3VZCIiIgmXnuPGXHp+NT5NOo2Fa05gQJgXTuRXoLKuoVB5QnwgVqflQyYAa54Zjh4B7vh//0vB+mMFCPfRYMv8kRCE1p3ITkREdDVa+/f3NRUoA4DRaMTy5ctx/PhxAEBcXBzuvPNOyOUsTHU21pGd/VnmbQUGhnthVLQ/3lqbjtVp+QCABweFomeguXj5nfv6IGHBJpwtrsbhc2VXfSo7ERGRPV1T2MnIyMCECRNw/vx59OxpnrZYuHAhQkJCsGrVKnTv3t2ujSTHstbsAICXRokPp/VDkM4Vnq4q/H15GnSuSswd3UO6R+uixMie/liVlod1R/MZdoiIyKGuqWbn6aefRvfu3ZGTk4MDBw7gwIEDyM7ORkREBJ5++ml7t5EcLNRbA5llJuqde/sgSGcOPw8OCsWmeSOw5plb4OOutnnPmLgAAMC6o/nt2lYiIqJLXdPITlJSEvbs2QNvb2/pNR8fH7zxxhsYOnSo3RpHHYOPuxrv3d8XSrkMt8UE2Fzr5ufe5HtGRftDKRdwuqgKGYWViPRv+j4iIqK2dk0jO2q1GhUVFZe9XllZCZVKdd2Noo5nct8umBAf1Or7tS5KJHT3BQCsP8bRHSIicpxrCjt33HEHnnjiCSQnJ0MURYiiiD179mD27Nm488477d1G6qTGSlNZBQ5uCRER3ciuKex8+OGH6N69OxISEuDi4gIXFxcMGTIEkZGReP/99+3cROqsbo8NgCAAh3JKkV9W6+jmEBHRDeqaanY8PT3x+++/IyMjQ1p6HhMTg8jISLs2jjo3fw8X3BTqhZSsi9hwLB8zEsId3SQiIroBtTrsXOk08y1btkjfv/vuu9feInIqY2IDkJJ1EeuOFjDsEBGRQ7Q67Bw8eLBV93G3XGrsthh/LFxzAvvOlsBkEiGT8f8fRETUvloddhqP3BC1VriPG+QyAXUGEwor6hCoc3F0k4iI6AZzTQXKRK2lkMukHZiziqsc3BoiIroRMexQmwvz0QAAskqqHdwSIiK6ETHsUJsL9TaHnexihh0iImp/DDvU5tprZKdWb8TLy49gwzFuYkhERA0YdqjNhXq7AQCy2zjsrDiUi//tycK/16e36ecQEVHnwrBDba5hGqttC5S3n7oAALhQWd+mn0NERJ0Lww61uVDLNNbFaj3Ka/Vt8hkmk4gdGRcsn1MPk0lsk88hIqLOh2GH2py7WgFfdxWAtitSPpJbhpIq84iO0SSiotYgXaupN+Ivvx7CpuOs5SEiuhE5NOxs27YNkyZNQnBwMARBwPLly22uz5o1C4Ig2HyNGzfO5p6SkhJMnz4dWq0Wnp6eePTRR1FZWdmOvaDWsE5lZTUKO3llNaisMzT3lquy7WSRzc8l1Q1TWdtOFeHn/efw1lrW8hAR3YgcGnaqqqrQp08fLFq0qNl7xo0bh7y8POnrhx9+sLk+ffp0HD16FBs2bMDKlSuxbds2PPHEE23ddLpKUt2OpUg5t7QGI9/eilHvbMWR82XX/fxtlnodK+soDwAUVtQBAM5cqITBaLruzyIios7lmk49t5fx48dj/PjxLd6jVqsRGBjY5LXjx49j7dq12LdvHwYMGAAA+OijjzBhwgS88847CA4ObvJ9dXV1qKurk34uLy+/xh5Qa4X6WFdkmYuUNxwrQJ3BhKKKOjzw2R4sfugm3BLld03PrqjV40DWRQCAr7sKFyrrcbFR2LlgCTt6o4iskmp093O/nq4QEVEn0+FrdrZu3Qp/f3/07NkTTz75JIqLi6Vru3fvhqenpxR0AGD06NGQyWRITk5u9pkLFy6ETqeTvkJCQtq0DwSEXTKNlWSZdtK5KlFZZ8AjX+3DysO51/Ts3aeLYTCJCPfRoFcXHQDbaaziqoZge6qAU5xERDeaDh12xo0bh2+//RabNm3Cm2++iaSkJIwfPx5GoxEAkJ+fD39/f5v3KBQKeHt7Iz8/v9nnvvjiiygrK5O+cnJy2rQf1GhjweJq1OqN2HXaPO30v0dvxqQ+wTCYRLy0/AiM17CKatspc3Aa3sMP3hpzIbTtyE7D9xmFFdfcByIi6pwcOo11JQ888ID0fXx8PHr37o3u3btj69atuO222675uWq1Gmq12h5NpFayLj/PK6vBzowLqNWbEKh1QXwXHd67rw+2pheitFqPtPNl6BviCcC8iuqpHw7ifGkNVAoZ1AoZ7hsQgnv6d7V5tnV/neFRfth9xjzy19zITkYhR3aIiG40HXpk51LdunWDr68vMjIyAACBgYEoLCy0ucdgMKCkpKTZOh9yDD93NVyVcphE4PvkbADAiB5+EAQBCrkMCd18AADbG62qWnMkDxuPF+B4XjkO5ZRib2YJ/vHHUegbFRlnFVchq7gaCpmAwd194O3WxMhOo00GTzHsEBHdcDpV2Dl37hyKi4sRFBQEAEhISEBpaSlSUlKkezZv3gyTyYRBgwY5qpnUBEEQpBVZW9LNAXVkz4aC5Ft6mL/fntGwqmrtEfNU5H0DuuKLmQPg7aZCRZ0BKZZiZADSOVgDw73hrlbAyzKNVVLVsHnhhUrbkZ1rmSojIqLOy6Fhp7KyEqmpqUhNTQUAZGZmIjU1FdnZ2aisrMTzzz+PPXv24OzZs9i0aRMmT56MyMhIjB07FgAQExODcePG4fHHH8fevXuxc+dOzJkzBw888ECzK7HIcaxTWaIIKGQChkb5StduiTR/fzD7IqrqDKiuN0hFzLOGRGB0bABGWALRlhMNo3nrj5rDzpi4AACAt5sSgHkXZcB8OKh1g0GZANQZTDh/sabN+khERB2PQ8PO/v370a9fP/Tr1w8AMG/ePPTr1w+vvPIK5HI5Dh8+jDvvvBM9evTAo48+iv79+2P79u029Tbff/89oqOjcdttt2HChAkYNmwYPvvsM0d1iVpgXZEFADeFeUHromy45qNBVy9X6I0ikjOLkZRehDqDCaHeGsQEeQAARkWbi9GtI0MXKuuwL6sEADAmzjxt6XVJgbJ1vx2lXECUv/k5GUUsUiYiupE4tEB55MiREMXmpxTWrVt3xWd4e3tjyZIl9mwWtRHryA4AaZTGShAE3BLlix/25mD7qQtSSBnXKxCCIJjfE+UHmQCcLKjEuYvV2HHqAkQRiO+iQxdPVwCQanasBcrFlnodHzc1ogLckV5QgVMFlbg1OqBtO0tERB1Gp6rZoc4ttNHITuN6HathkebXtqYXYfNx8+jNuF4NheY6jRL9w7wAAFvSi7DuqLmmZ2xcQ3DxsoSdsho9DEaTVK/j466SRnZYpExEdGNh2KF2ExushYtShu5+bogN0l52fUh3HwgCkHmhChV1BgRo1ejb1dPmHutU1opDudiZYV5mPjauIRB5upqnxkTRHHisYcfX3TyyAzDsEBHdaBh2qN34e7hg7TPD8cMTg6Wpqca83FTobdkBGTCHGJnM9r5RPc1hZ29mCeqNJnTzdUOkf8PxDwq5DDrXhiJl67Jz88iO+b7ThZUtTp8SEZFzYdihdhXu6wZ/D5dmrw9rtEJrXNzleyVFB3ogSNfw/jFxgZcFJx+3huXnxZaRHT93NcJ83CCXCaisMyC/vPa6+kFERJ0Hww51KCN6mEduvN1UuDnC+7LrgiBIU1lAw5LzxryksFNnU7OjUsgQbimS5hlZREQ3DoYd6lBujvDGW/f0xucz+0Mhb/r/nrdZwk6g1uWymh4ANhsLFlc1rMYCwCJlIqIbUIc+G4tuTPcNaPkU+luj/fHa5Dj06qK7rKYHsN1YsKjCUqDsYQk7Ae5Ye5QHghIR3UgYdqjTEQQBMxPCm73eMI1V32hkx/yatZj5WG552zaSiIg6DE5jkdPxtkxjFVfWSZsT+llGdqx1QIfPl9kcFkpERM6LYYecjnVkJ/NClXTop3Vn5SCdK6IDPSCKwLZTRc0+g4iInAfDDjkd68iOtQjZU6OEslGx8wjL7s1J6VcOO7V6I95YcwJHzpe1QUuJiKg9MOyQ07GO7FTXGwE01OtYWc/lSjpZBJOp5c0Ff005h0+STmPhmuNt0FIiImoPDDvkdLwvCTc+7mqbnweEecNNJUdxVT2O5LY8YnMopxQAkJ7PpepERJ0Vww45Hes0lpXfJWFHpZBhaKR5p+atV5jKSrNMX12orENZtd6OrSQiovbCsENOx8NFAXmj/Xd83FWX3TPScsbW1vTCZp9TqzfabD6YUcS9eYiIOiOGHXI6MpkAL41S+tn3kpEdoKFIOTWnFKXV9ThdVImFa45L01YAcDyvXFrNBQAZ3HWZiKhT4qaC5JS8NCqbE88v1cXTFVH+7jhVWInEJQeQfKYEBpOIpPQirH12OABctgLr0vO0LlbVQ+uqtBlFIiKijocjO+SUvBoVKTc1sgMAIy2jOzszimGwjOCcyK/AuYvVABrqdawbEmYUNYSd3aeLcdM/N+CttSfs33giIrIrhh1ySo2LlH2bGNkBgCn9ukKtkCHS3x3f/Olm3Bxu3l1503FzHU/aefOREnf2CQZgO4218nAuRBFYsjcb9QZTm/SBiIjsg2GHnFLjkR3rieeXig3W4tD/jcGGucMxoocfbo0xFy1vOlFoLk4uMBckT+nXBQBwvrQGNZa9e5IzSwAAFbUG7MjgTsxERB0Zww45JevJ50DDiedNcVHKIQjmmpvRlrCz53QxDmRdhMEkwttNhbhgLbw0SogicLqoEhcq6y4Z5cm7Ynuyi6sx7bM92JVx4Vq7RERE14hhh5ySl2UaS62QwU0lb9V7uvu5I9Rbg3qjCYuTTgMAenXRQRAE6bT000WV2GsZ1dFYnrvhaAHqDMYWn/3T/mzsPlOMN1jjQ0TU7hh2yClZd1H2dVdLIzdXIggCbrOM7mw/ZR6Bie+iBQAp7JwqqETymWIAwD39uyJAq0ZFnQHbT7Y8YnPSspLr8LkyZBdXX2VviIjoejDskFMK83EDAET4ul3V+0bHBNj8HN9FB8A86gOYi5T3nDGP7CR088GE+CAAwKq0lqeybKa90nKvqk1ERHR9GHbIKd0U6omvZg3E2/f2vqr3DQz3hoe6YfupXpawExXgAQA4kH0R6ZbC5ZsjvHFHb3PY2XisALX6pqeyavVGZBVXST+vPHTlGh8iIrIfhh1ySoIgYFS0P4J0rlf1PpVChuGWU9G9NEp08TS/3zqNVVhRBwCI8neHj7sa/UK8EKRzMU9lnWp6KutMURVMIuCmkkMhE3AsrxxnirgbMxFRe2HYIbqEdWpqcDcfqd4nWOciFSQDwKBu5j15ZDJBuv/DTadQUXv5YaGnCs0jQTFBWukA0lWtWMFFRET2wbBDdIkJ8YH45k8345939ZJeEwRBqtsBgEERPtL3DyeEw1OjRNr5Mjzy1T5U1Rlsnmc9ZiIqwAMTLdNerVmuTkRE9sGwQ3QJQRAwoocffC45ZsI6lQU0jOwAQKiPBt89OghaFwX2Z13Eo9/skzYfBICTlhqfKH93jI0NhFIuIL2gQtq0kIiI2hbDDlErWcNONz83+Hu42Fzr1UWHbx8dBHe1AnvOlODNRvvpWFdi9QjwgE6jxPAoc00QR3eIiNoHww5RK43vFYhgnQseGRLe5PW+IZ54974+AIAVh3JhNImo1Rtx1rISKyrAHJZGRZv38jmYU2rz/iPnyzDxw+3YdpLHTxAR2RPDDlErdfNzx64Xb8OMhPBm7xkV7Q+dqxLFVfXYf7YEmRfMK7G0Lgr4W46tiAkyb1R4Iq/c5r0/7svG0dxy/GPFUZgsp7ATEdH1Y9ghsiOlXCbtwrzuaAFOFTYUJ1tXdvUMNO/ZU1hRh5Kqeum9R3PN4ed0URWSTnF0h4jIXhh2iOxsXFwgAGDd0XyczDcXIfcIaChudlcrEOqtAQCcyDcHHKNJxIm8hoLl/+7ItEtbjueVY97Pqcgp4REVRHTjYtghsrPhPfzgqpTjfGkNVhw2Hw0R6e9hc491dMcacDIvVKJGb4RKIYNMMJ/NdbKZ1Vp6o6nVbfl462ksPXAev6Scu5auEBE5BYYdIjtzUcoxsqd5xVWW5dDPxiM7ABBjCTvplpEf6xRWr2AtxsSaR4aaGt35NeUcol9eiz8Ote58rUOWIuiiitqr7AURkfNg2CFqA2MtU1lWUZeM7ERbi5Qt01jWsBMXrMOjt0QAAJYePI/iyjrpPVV1BixcfRxGk4i1R668bL2kqh7ZlumrC5X1V7ibiMh5MewQtYFR0f5Qys0FyR4uCgRobTcotE5jpRdUwGgScTS3DAAQF6zFgDAv9O6qQ73BhMVbT0vv+XrXWRRbCpqP5dqu5GrK4XOl0vcXGoUmIqIbDcMOURvQuSqR0N18DlaUv7u0Essq3McNaoUMtXoTsoqrpPASF6yDIAh4+tYoAMAXOzKx9kg+ymr0+DSpIfhklVSjstGxFLmlNfjXqmMoqmgINYdyyqTvizmyQ0Q3MIYdojbywMAQAMAwy47JjcllgjS6syW9CBer9VDIBGnjwdGxAXh0mHk6a/4vh/DqH0dRXmtAlL87/D3UEEUgPb9hdOf9jSfx+fZMvLMuXXqNIztERGYMO0RtZEJ8ELb/ZRSeujWyyes9A8xhZ+kB80qpSH93uCgbTlb/6/ho3Bzhjco6A5YdPA8AmHd7D8QFm+t9Gk9lJWeWAAA2HC+AwWiCKIo41CjsVNcbUV1ve0ApEdGNgmGHqA2FeGuglDf9r5m1SNlanBxrCTFWSrkMix68Sar3iQvWYlyvQOm+Y5Zl64XltdKqr5Kqeuw9W4LcslpcqKyHQiZAZfl8e01lVdYZkHymmLs8E1GnwbBD5CDRgbYrtOKCdZfd4+ehxpcPD8SY2AC8ObU3BEFAbJD5vmOW4yb2ni2xec/aI/nSkvOegR7wsxxTUWSnqazXVxzD/Z/twfpj+XZ5HhFRW2PYIXKQy8OOtsn7enXR4bOZA9CriznkxARZ9+gph9EkYv/ZiwDMp7ED5rCTagk7fUI84euuAgBcqLBP2DliWTl2MLvULs8jImprDDtEDuLjrpZGXYDLp7GaE+bjBo1Kjlq9CZkXqrDXUq/z1K2R8FArUFhRh18tOyb36aqDr7v5M4qr7DONZT16IsNy7hcRUUfHsEPkQNbRnVBvDbQuyla9Ry4TpPclZxbjuGVV1tDuvtIhpNYDRs0jO+awY4+RnbJqPcprzYXOpxh2iKiTYNghciBraGluCqs5MZbi5u/2ZEMUgTAfDfy1LhjXK0i6x1UpR6SfO3ws01j2GNnJuVht832t3njdzyQiamsODTvbtm3DpEmTEBwcDEEQsHz5cpvroijilVdeQVBQEFxdXTF69GicOnXK5p6SkhJMnz4dWq0Wnp6eePTRR1FZyf/ipM5hZkI4RscE4MmR3a/qfdYpr+OWIuUBYd4AgBGWQ0gBIL6LDgq5TBrZsUeBcuPT00UROF3Ef9eIqONzaNipqqpCnz59sGjRoiavv/XWW/jwww/xySefIDk5GW5ubhg7dixqaxsONZw+fTqOHj2KDRs2YOXKldi2bRueeOKJ9uoC0XUJ8dbgi4cHoHdXz6t6X2yQ7UjQzRFeAABXlRyjos2bGPYNNT/Tx44Fyo1HdgDW7RBR56Bw5IePHz8e48ePb/KaKIp4//338dJLL2Hy5MkAgG+//RYBAQFYvnw5HnjgARw/fhxr167Fvn37MGDAAADARx99hAkTJuCdd95BcHBwu/WFqD31DPSAIJhHVwBgYLi3dO3vE2MRrHPF/xveDQDgZ8cC5ewShh0i6nw6bM1OZmYm8vPzMXr0aOk1nU6HQYMGYffu3QCA3bt3w9PTUwo6ADB69GjIZDIkJyc3++y6ujqUl5fbfBF1JhqVAhG+5qXmvu4q6XsA6OLpipfuiIWPJeT4WlZ82ePIiJySGgBAD8uxFgw7RNQZdNiwk59v3rAsICDA5vWAgADpWn5+Pvz9/W2uKxQKeHt7S/c0ZeHChdDpdNJXSEiInVtP1PasU1kDw70vO2i0MR838zRWabUeeqMJAFBWo8e6o/moN5iu6jOt01ijos3/3jUOO0uSszHu/W04wzoeIupgOmzYaUsvvvgiysrKpK+cnBxHN4noqt3Tvyt83VV4cFBoi/d5aVSQy8xhyLok/d316fh//0vBsz8dbPWxDyaTiHOWkZ1be5rDTuaFKuiNJphMIt7feBIn8ivw4aZTLT2GiKjdddiwExgYCAAoKCiweb2goEC6FhgYiMLCQpvrBoMBJSUl0j1NUavV0Gq1Nl9Enc3Inv7Y/9LtuKWJU9Ubk8kEeFtGd4osRcr7LLsur07Lx3sbT7bq8wor6lBvNEEuE3BTmBc0KjkMJhFZxdXYn3URhZZnrzych7yymmvtFhGR3XXYsBMREYHAwEBs2rRJeq28vBzJyclISEgAACQkJKC0tBQpKSnSPZs3b4bJZMKgQYPavc1EHZV1Kqu4qh51BiNOFlRI1z7anIFlB89d8RnWKaxgTxco5TJE+lvrdiqwOi1Pus9gEvHNrix7Np+I6Lo4NOxUVlYiNTUVqampAMxFyampqcjOzoYgCHj22Wfxz3/+E3/88QfS0tIwc+ZMBAcH46677gIAxMTEYNy4cXj88cexd+9e7Ny5E3PmzMEDDzzAlVhEjViPpbhQUYdTBZUwmEToXJWYPcK8v88Lv6bhyPmyFp+RbTlZPdRbAwCI9DOHnZMFlVhzxBx2Hhhorn9bkpyFqjrDZc+o1Ru5Nw8RtTuHhp39+/ejX79+6NevHwBg3rx56NevH1555RUAwF/+8hc89dRTeOKJJzBw4EBUVlZi7dq1cHFxkZ7x/fffIzo6GrfddhsmTJiAYcOG4bPPPnNIf4g6KuvIzoXKOhy1HOTZq4sWfxnbE7dF+6PeaMKn2860+AzryE6IlyXsWFZk/XbgHArK6+ChVuDVO+MQ7qNBea1BOp/LymQSMeurvbjt30k4mH3Rrv0jImqJQ/fZGTlyJESx+eJIQRDw2muv4bXXXmv2Hm9vbyxZsqQtmkfkNBofBnruormeJi5YB5lMwNzbe2DTiUKsO5KP4so6acn6pazLzkMsIztR/uajLrIsIz6jYwPgopTj0WERePn3o/jvzkw8NDhMKo7+aX8O9pwxH1q650wJ+oV6tVFviYhsddiaHSKyH99G01jWkR3reVy9uujQu6sO9UYTlh443+wzrEdFWMOOtWbHanwv86KAqf27QueqRFZxNf69Ph2iKOJCZR3eWHNCuvdUo5ohIqK2xrBDdAOwTmMVVtTheJ45aMQF66Tr0242L1//YW92s6OtDdNYrtL/qhTmXyHuagWG9zCvCtOoFJg/pgcA4OOtp/HK70fxz5XHUFajl87tOlnIsENE7Ydhh+gGYB3ZOZh9ETV6I1yVcptdlyf1CYZGJceZC1VIziyBKIpYtCUDEz/cjsPnSlFnMCK/3HwmnXVkRyGXoZvlGbfF+MPFEmQAYEZCOF6/qxcEAfjfniwsT82FIABvTI0HYN6MsLX7+xARXS+GHaIbgPV8rKp6IwAgJshDqqUBzCMzk/uaVzB+u/ssnvv5EN5el46jueV49sdUnC6sgigCGpVcGiUCzDspy2UCHrz58o0NZwwOwwcP9IPC8jkzB4fhjt7BUClkqNWbLjtUlIiorTi0QJmI2of15HOrXl10l90z7eZQ/LA3B6vTzEetyGUCPFwUOHOhCi8uPQzAvBKr8dEU88f0xOwR3aFzVTb5uXf2CUawzgXJmSV4ZGg45DIBkX7uOJZXjvT8CoT5uDX5PiIie+LIDtENwMfNdoWVtTi5sfguOul1D7UCXz8yEP++tw8A4NA5c1FziLerzXvkMqHZoGM1INwbiaMioVGZ/9vKeojoKR4iSkTthCM7RDcAlUIGrYsC5bXmjf4aFydbCYKAV++Mw3d7svDnkZHoGWheWn5X32AsT80F0FCvcz2iAszPPckVWUTUThh2iG4Qvh5qlNcaoJQLiApwb/KegeHeGBjubfPa/02Kw46MYlyorEO4HaadekhhhyM7RNQ+OI1FdIOwbiwY5e8BtUJ+hbsbeLmp8MXDAzBjcBju6tflutthncY6XVQJg9F0Tc/YlXEBH2/N4IouImoVjuwQ3SB8LUXKTdXrXEnfEE/0DfG0SztCvDRwUZpXZGWVVKO7X9OjTM0RRRHP/pSKwoo6RPq5Y0xcoF3aRUTOiyM7RDeIAWHm6anbYvwd2g6ZTJCOmriWnZRPFlSisKIOALAlvciubSMi58SwQ3SD+NOwCBz6vzEY1yvI0U2RaoasdTs7My7gjTUnUF6rt7mvsKIWe84U27y2I+OC9H1SemGL5+sREQEMO0Q3lCstE28vPRqtyDpZUIE/fb0PnySdRuL3B6C31PGcvVCFiR/uwAOf7cGuRgFnZ6Pvc8tquYSdiK6IYYeI2p21SPlobjmeWnIQdQZzwNl+6gJeXn4EuaU1mP5FMoos01U/7ssBAOiNJmmkp4unec+fremF7d18IupkGHaIqN1Za3YyL1QhvaACvu5qvDW1N2SCOdiM/2A7zpfWwN9ypte6o/koq9EjNacU1fVG+Lip8KdhEQCArXaq21l5OBefbzvDaTEiJ8SwQ0TtrounKzSqhuXv797XB/cNDMErd8QCAMpq9Oji6YrliUPRM8ADdQYTVh7OxY5T5imsIZG+GNXTfMr6vrMlqKwzXFd7iivrMPenVPxr9XGknS+7rmcRUcfDsENE7U4mE9DLsovzE8O7YXgPc3CZNTQCfxnXEwndfPDdY4MQ7OmKe/p3BQD8sv+cVK8zLNIHEb5uCPXWQG8Usft0cdMfZCGKolQL1JTfU3OhN5pHdC4tiCaizo9hh4gcYsHdvfDG3fF4fmxPm9f/PDISPzwxGBG+5t2aJ/cLhlwmIDWnFAeyLwIAhkb6QhAEjLSM7rRUtyOKImZ9tQ83/2sj9p0tafKeX1POSd/vOdP0PUTUeTHsEJFDRPp74IGbQ6GUt/xryN/DBSMtIz8mEQj30aCrl/mMroawU9Rsrc2+sxeRdLIIF6v1mPnlXpvVXABwNLcMx/LKG+7PLLnmnZ2JqGNi2CGiDu/eAV2l74dG+krfJ3TzhUohw/nSGhzNLW/qrfhi+xkAgJtKjhq9EY98vQ+bTxRI139LOQ8AGBsXAA8XBSrqDDbhh4g6P4YdIurwbo0OgJfGvEfQsEZhx1Ulx5jYAADA+xtPXfa+sxeqsOG4Odj8PDsBY2IDUG8w4YlvU/Dj3mzUG0xYnmoOO/cPDMGgCPMu0/aq29lwrABrj+Tb5VlEdO0Ydoiow1MpZPhwWj88c1vUZWdhzb29B+QyARuPFyAly7be5qudmRBFYFRPP8QF67Bo+k2Y0q8LDCYRf12ahke+3ouSqnr4uqsxPMoPg7v5AGi+bufNtSdw27+3YvhbWzBk4SY89s3+ZgufS6rqMfu7FDz5fQrOXay2w58CEV0rhh0i6hRuifKTgk1j3f3cca9lxdaba9Kl2p3S6nr8vN9cePzYLd0AAEq5DO/e1wdzR/cAAOzMMI/g3H1TFyjkMinsNFW3U1Grx+Ktp3G6qArZJdXILavFxuMF2HCsAE3Zm1kCo0mEKAJr0ji6Q+RIDDtE1Ok9MzoKKoUMe8+WYOvJIuiNJny27Qxq9EbEBGkxpLuPdK8gCHhmdBQ+nNYPKoUMMgFSWIoJ0jZbt5Oebz601M9Djd+eTMDMhDAAwNe7zjbZpr2ZDaNDq9Ly7NldIrpKCkc3gIjoegXpXDFrSDg+23YG838+hHqDCRWWjQYfGxYBQRAue8+dfYLRK1iLi9X1iLKc1SWXCRgU4Y2NxwuRfKYEvbt6Svcft4SfXsFa9A/zRhdPDZYkZ2NvZgmO55UjJkhr8/y9ZxvqflJzSnHuYrW0ioyI2hdHdojIKTw5ojs81AoUV9Wjos4AbzcVHhsWgcl9g5t9Tzc/d/QP87Z5raFux7ZI+bhlZCfaEmoCdS4Y28tcP/Tt7rM291bU6nHMsjqsu595vyBOZRE5DsMOETkFLzcVvnpkIP46PhrLE4di399H46U7YqG4wj4+l7KGHWvNjZV1ZKfxCM6sIeEAgGUHz6O0ul56PSXrIkwiEOajwcwE8z2cyiJyHIYdInIaA8K9MXtEd/QN8byskLm1GtftHLGck2UyiVLNTmyQR8PnhXkhJkiLWr0JP+/PkV631uvcHO6N8b0CIQjmqazzpTU2n2UyiXj1j6N4d316k23hoaRE9sGwQ0TUiFwmIMEyurPztHm35eySalTXG6FSyBDu4ybdKwgCZg0xFyp/uztLWoYuhZ0Ib/hrXTAw3DxVtuaS0Z2Nxwvw9a6z+HBzBrKLG5an19Qbcfu7Sbj3k90tnunVnMYjUkTEsENEdBnrLs3WoyWsU1g9Azwumxab3LcLfNxUOHexBp8mnUat3ohD50oBAIMizKFpYnwQAGDlYduw8+m2M9L3mxrt6rw1vRCnCiuxP+sivt+TdVVtL6yoxdA3NuPxb/df1fuInBnDDhHRJaxhZ9/Zi6jVGxuKkwM9LrvXRSnHy3fEAgA+3JSBX1LOQW8UEaBVI8TbFQAwvlcgZJaprHVH8y3PLkFK1kXpOZtPNBxmuvZoQzHz+5tOoaxa3+q2f7cnG/nltdh0vAC1emOr30fkzBh2iIgu0d3PDYFaF9QbTEjJuthkcXJjk/sGY2RPP9QbTXj1j6MAgJsjfKQl7/5aFzw+3Lyx4V9/O4zC8lp8mnQaADC4W8MRFZV1BtQZjNh83Bx8vN1UKK3W44NNlx+F0ZR6gwlLkrMBmA9NzSisvJbuEzkdhh0ioksIgoAhkeYpqB0ZF3Ai3xx2ooMuH9mx3v+vKfHQqORSvczNEbZL2ufd3gOxQVpcrNbjsW/3Y+PxQggCsGBKPMJ9NNAbRWw/WYRdGcWoqDMgQKvGe/f3BWBe2n6m6MrBZXVaHi5U1kk/W4uqiW50DDtERE2wHji6/mg+ckrMq6hiApse2QGALp6u+MvYntLPgy4JO2qFHB880BdqhQyHz5lXeY2NDUQ3P3fcFmM+zHTTiULp4NCxcYEY0cMPo3r6wWASMf+XQ0jJKmlxhZZ1N2eNSg4ASC9g2CECGHaIiJpkrds5XVQFAAjUusDLTdXie2YkhGPazSGYdnMIovzdL7seFeCBF8dHSz//vxHmqa3bov0BAFtOFEqntI+zHHj694mxUClkOJBdiqmLd2Pc+9svW9UFAIdySpGaUwqVXIYnR3QHAJy4ypGdeoMJLy5Nw/KD56/qfUQdHY+LICJqQoDWBZH+7lLdS0wzU1iNyWUCFt7du8V7Hh4SjpKqerio5OgX6gXAvD+QdfdnAPDSKKVpsEh/d/yeOBT/3ZGJFYdzkV5Qgbk/p2JUtD9clHLpud9YRnXu6B2EIZE++PcG4ORVhp3NJwrxw95srDiUi3G9Am2eT9SZcWSHiKgZ1qksoOGYiOslCALmjemJP4+MlF5TKWQY3sNP+vn22ACbJe4xQVq8fW8fJP9tNHzd1ajVm3Awu1S6XlJVLy1rnzkkHD0sZ33ll9de1Uou65L5yjoDtqYXtnwzUSfCsENE1IzGp6U3txLLXm6L8Ze+H2c5c+tSOlclEixt2t3o7K6t6YWoN5oQHeiBviGe8HBRoounedn71dTtHMoplb5fcah9j7c4fK4UlZbDW4nsjWGHiKgZg7v7SMdOxLZx2BnZ0x8eLgoEaNVSvVBTrLs77zndEHase/Q0Dkw9LXsCpVtWkgHAgeyLSDpZhPOlNZcVOptMolQ4DZh3d26v8LElvRB3/mcn/rnyWLt8Ht14WLNDRNQMrYsSb03tjeKqOkQ2UXBsT95uKqx55hYo5TKoFc3XylhHdg7mXERNvRFKuYBtJ4sAALdGN4SdHgEe2HyiUBrZOXyuFFMX74I142hUcvxpaATmW1aQnS6qRGWdAa5KOQK0apwtrsbGYwW4q1+XtuiuDWtw23W6+Ap3El0bjuwQEbVgav+ueGJ493b5rK5eGgRoXVq8J9xHg0CtC/RGESlZF3EwpxTltQZ4apToG+Il3RctjeyYw853e7IgioDWRQGFTEB1vRGfbjuN8lpzTU+qZQorvqsOd/Y1B5wVh3Lt3cUmWVeNZZdUo6L2yjVGtXojvth+BnllNVe8lwhg2CEi6lQEQWhUt3NBmsIa0cPP5qR3a5Fyen4Fymr0Ug3Ol7MG4vjr4xDl7w69UZR2a7YWJ/cN8cSk3uazvLadKkJpdX2b9+lEo6m21iyX/+/OTPxz1XG8toLTXtQ6DDtERJ2MtW5n9+libLGEnVE9/W3u6e7vBrlMQHmtAZ8mnUaN3ogof3cMCPOCUi7DeEsR9GrLnj3WkZ0+XT0RFeCB6EAP6I2idJbXtarVG7FoSwaOnC9r8nppdT0Kyht2fbYezdESa0DbkXEBhms4FZ5uPAw7RESdjHVkJzWnFCfyKyAI5pGdxtQKOSJ83QAAX2zPBAA8OChUOq9rXC/z6E3SySKUVNXjRJ55RKVPiA4AMKlPMADgj+ucynp/4ym8vS4df1uW1uT1S0dyjuW2HHbKqvU4kG0+QLWi1oC0ZkIUUWMMO0REnUyItwZdPF1hOYYL/UI8m9zd2boiq95oglohw939ukrXYoI8EOajQZ3BhEVbMmAwifB1V0tL1u+0hJ1dp4uRW3pttTE5JdX47w5z0Eo7X4aSqsunxE5YRnLUCvNfR1ca2dmeUST1GwB2Zly4prbRjYVhh4ioE0potAfQpVNYVtEBDbs+39E7GDqNUvpZEARpP5//7c4CAPQN0UkjPyHeGgzu5g1RBJY1c3xEdb0B64/mY+2RPOw+XYwT+eU200pvrj2BesvPoth0MLGuFhsdaz4f7ER+RYtTU1vTzSvPfN3N4W4Hww61AsMOEVEn1HjDw1HRTYedHoENYefBQaGXXR9vmcqyBpI+XT1trt/TPwQA8GvKOZt9eXZlXMAzPx5E/9c34on/pWD2dwcw7fM9GPf+dox5fxv2nClGStZFrDycB0GAtDv09lNFl7XBOo01JjYAGpUcdQYTzhZXNdkfk0lEkmWZ/bOjewAADmSVorqemxFSyzp02Hn11VchCILNV3R0wyF6tbW1SExMhI+PD9zd3TF16lQUFBQ4sMVERO1jWJQvPNQK9AhwR1xw0xseDgjzgs5ViSHdfXBTqOdl1/t01SFI17DUvU+I7T3jewVCo5Ij80KVVCez7OA5PPhFMn5PzUWN3ogQb1f0D/NCdz83aFRynCmqwgOf7cHj3+4HANzXPwSPDYsAAGw/dcEmNJlMonR+V2yQVloufyyv6RVZx/LKUVRRB41KjnsHdEUXT1fUG03Yd/ZiK/7E6EbW4TcVjIuLw8aNG6WfFYqGJs+dOxerVq3CL7/8Ap1Ohzlz5uDuu+/Gzp07HdFUIqJ24+/hgo3PjYBaIZOmni7l465G8t9ug8zyH4uXEgQBY+MC8bXlENFLR3bc1ApMiA/Crynn8GvKOfi5u+Dl5UcBmGt6Hhkajr4hntKzy6r1eHPdCSxJzkZJVT00KjmeG9MDWlclVAoZ8spqcbqoEpH+5lBz7mINquqNUMlliPB1Q0yQFgeyS3Est1yqGWrMOqozpLsP1Ao5hkb64Of957Az48JlBdpEjXX4sKNQKBAYePk5MWVlZfjyyy+xZMkS3HrrrQCAr776CjExMdizZw8GDx7c3k0lImpXV9qAEMAVTy6/s28wvt51FnHBWpuaHqt7+nfFrynnsPJQHo7nVaCyzoCB4V547/6+Nvv6AIBOo8SCKfGYelNXfL7tDCb1CYa/pY03h3tjR8YFbD91QQo71v11Iv3doZDLpPPHmitSth5OOsJSozQ00hc/7z+HHadYt0Mt69DTWABw6tQpBAcHo1u3bpg+fTqys7MBACkpKdDr9Rg9erR0b3R0NEJDQ7F79+4Wn1lXV4fy8nKbLyKiG9FNoV74ZXYCPp3Rv8nrN4d7I8TbFRV1BqTmlMJDrWgy6DTWP8wLn8zoj4mWzQkB4JYo83lf2xsFE+vuztbpq9jg5sNOWY0eBywnvY+0jOIM6W5+5rG88iZXehFZdeiwM2jQIHz99ddYu3YtFi9ejMzMTNxyyy2oqKhAfn4+VCoVPD09bd4TEBCA/PyWN8FauHAhdDqd9BUSEtKGvSAi6tgGhnujq5emyWsymYCpNzUsWf/nlF7N3tuSYZaws+dMMeoN5oJoa3FydJA57EQHekAQgMKKOlyobNhosLregDfXnoDRJKK7nxtCvM2f7+ehloLSrtMc3aHmdeiwM378eNx7773o3bs3xo4di9WrV6O0tBQ///zzdT33xRdfRFlZmfSVk5NjpxYTETmfB28ORc8ADzw2LAKT+17bwaAxgVr4uqtQXW+Uip2t01g9A80jOhqVAhE+5o0Qj+eVQxRFbD5RgNvf3YYlyeZR/YcGh9k8d5jlhPiNx2wXp4ii+ewwPXdYJnTwsHMpT09P9OjRAxkZGQgMDER9fT1KS0tt7ikoKGiyxqcxtVoNrVZr80VERE3z17pg3dzheOmO2Gt+hkwmSMHk5/05KKmqR+YF8xLz6EZL5K11Ox9uOoURb2/Fn77ej/OlNeji6Yr/zhqAR4ZG2Dz3Dksh88rDeTYHg3689TSmLt6Fvy1teufmS2UUVmLB6uPYccmKMXIOnSrsVFZW4vTp0wgKCkL//v2hVCqxadMm6Xp6ejqys7ORkJDgwFYSEVFTbosxbxy49MB5DF64CSYR8NIo4e+hlu6JsUxp7Tt7Edkl1XBRyvDE8G7YMG84bo0OuOyZfUM8MSjCGwaTKO3WXFRRh0VbMgAAv6Scw/6zJS22a+mBc5j00Q58tu0MHvoyGRM+3IHfUs7BZGr70FNeq8eWE4UMWG2sQ4ed+fPnIykpCWfPnsWuXbswZcoUyOVyTJs2DTqdDo8++ijmzZuHLVu2ICUlBY888ggSEhK4EouIqAO6o3cQXp0Ui26+blLdTlywzmZZ/JSbuqJviCcmxgdh0YM34cDLt+NvE2KgUTW/eHj2iO4AgCXJ2Sir0ePDTadQXW+EtYb6ld+PwthEcKnVG/GXXw9h3s+HUKM3IiZIC41KjuN55Xjul0NYuOa4HXvftBd+PYxHvt6Hn/axnKItdeil5+fOncO0adNQXFwMPz8/DBs2DHv27IGfn7kS/7333oNMJsPUqVNRV1eHsWPH4uOPP3Zwq4mIqCmCIGDW0Ag8PCQcezNLkHSySDpw1KqLpyuWJw69queO7OmHngEeSC+owL9WHcNvB8zHW/znwZvw198O41heOX7Ym31Zvc8ba07g5/3nIBPMOzInjopEZa0BX+3KxPsbT+HLHZm4o3fwZZstAkBheS1WpeXBXa1AmI8buvu5wcddfdl9LSkor8V6S63R8tTzeODmy3e5JvsQRI6doby8HDqdDmVlZazfISLqhH5LOYfnfjkk/XxrtD/+O2sgvt6ZiVdXHIOnRoktz42UDkzNKq7C6HeToDeK+HRGf4yNs631fObHg/g9NRfRgR5Y8dQwKOUNEyFniirx0BfJyC2rlV5TygW8c28fmwLu31PPY/HW05h3ew+Mibu8lnTRlgy8vS4dACATgOS/jYafx9UFphtda//+7tDTWERERK1xZ99gBFuOvpAJwAvjzEcLPTQ4DNGBHiit1uOF3w5L01lvr0uH3ihiRA+/y4IOALxyRyy8NEqcyK/A59vPSK8fyy3HfZ/uRm5ZLUK9NRgW6YtgnQv0RhGv/H5UWjKfXVyNF347jBP5FXjy+wNYcSjX5vkmk4if95unrpRyASYRWHe05W1T6Nox7BARUaenlMvw5KhIAMC0m0PR07LCSyGXYcHd8VDJZVh/rACvrzyGQzml0iGl1lB0KR93NV6aaF599sHGU3ju50OYs+QAHvhsNy5U1iM2SIulfx6C7x4bhG1/GYWYIC3KavT416rjEEURf1uWhlq9CR5qBYwmEc/8eBC/ppyTnp+cWYKs4mq4qxX480hzu9ccyWvLP6I299zPhzDu/W2oqNU7uimXYdghIiKn8NCgUKx99ha8NrmXzes3hXrh3/f1AQB8vessHrMcUjqlbxdp1+am3H1TF9wS5Ys6gwm/HTiHlYfzUF5rQP8wL/zwxGD4Wmp0FHIZFt4dD0EAlh08jxd+O4wdGRegVsiwfM5QTLs5FCYRmP/LISzakgGTScRP+8z7Bk3qEyxt2rjnTAmKG22meDV+3JuNGV8m2yy/b08XKuvw24FzOJFfgW0nO94Gjx26QJmIiKi1BEFAdGDT4WVSn2DkldVgweoTKKqog0ouw7wxPa74vPfv74tfLCMyaoUM3m4qjI0LvOzMsb4hnpg5OAzf7M7Cz/vN98+9vQe6+7ljwZRecFXK8d+dmXh7XTpSsi5iZ4Y5EDwwMAShPhr06qLFkfPlWH+sANOuslA57VwZXlp+BAaTiL8vO4IvHx7Q7OGwbcXaHwDYfeaCzVEhHQFHdoiI6Ibw+C3d8MjQcPP3wyNadeyFj7sas0d0x+wR3fHIUPMO0s0drvrc2J4I0JpHe3p10eKxYeYNEAVBwCuTYvHG3fFQKWTYfKIQdQYTogM90LurDgAwvpc5HKxOa3kqq1ZvxJHzZdK+PLV6I577JRUGSy3S5hOFWJ3W/rU/jQ9j3XW6uN0//0oYdoiI6IYgCAJeuSMWW+aPxHO397T787UuSnw07SaMjgnA+/f3g0Ju+1fsAzeH4tfZCeji6QoAmJkQLo3ATIg3h51dp4vxe+p5rDiUi12nL9hsbKg3mjD9i2Tc8dEOPPh5Mk4XVeK9jSdxsqASvu4qPJxgXlr/6oqjKKtpv7oZURSxo9HIzpmiKhSU17bwjvbHpefg0nMiImo/FbV6nCyowE2hXjbTTeM/2H7Zie/Tbg7Fgim9IAgCFq4+jk+3NawMU8llMJhMMInAZzP6Y3gPP0z4YDvOXKjC9EGh+NeUeJtn6Y0mVNUZ4KlRtaqdBqMJtQYT3NUtV7xkFFZi9LtJUClkCPPW4FRhJT54oK+0DF9vNNks3bcnLj0nIiLqgDxclOgf5n1ZXc3fJkRjaKQPBkV4Y1CEN2QC8MPebPxjxTFsPlEgBZ1/3BmHkT39UG80B52pN3XFGEsd0YK7zQHn++RsHLQcuAqYR1+e/C4FA/+18bKpsuziamw5UXhZO2d/l4LBCzYho7Cixf7sOFUEABgY7oWRPc2b/u7KaJjKmv/LIcz+Xwqyi6tb+0dkdyxQJiIi6gBuifLDLVF+0s+/pZzD/F8P4etdZ/F9chYAYNaQcDw8JBwzE8Kw/lgB0s6VYfbI7tJ7BnfzwdSbuuK3A+ewcPUJ/PT/BkMQBGw/dQEbj5sDzTM/HoSbWoERPfyw9kg+5v2ciup6I75/bBCGWg5rPV9aI93/0eYMfPBAv2bbbZ3CGhbph56B7vh8eyZ2nzGHncPnSvF7qnmPoTm3Rtrrj+qqcWSHiIioA5ravyv+dZd5pEZvFNGrixYvTjDvCyQIAsbGBWL+2J6XTTPNH9sDaoUMe8+WYOPxQphMIt5YcwIA4OOmgt4oYvb/UvD3ZWmY/V0KquuNAGCzD9CaRqM/Kw7lSifUX0pvNGHPGfNBq7dE+WJguDfkMgHZJdU4d7EaC1abzxeb0q8LenXR2eOP5Zow7BAREXVQDw4Kxb/v7YPbYwPw8YP9oVY0vRKssSCdKx61rAR7Y81xLDt4HsfyyuGhVmD1M7dgeA8/1OiN+D7ZvNfPaMtp9GuP5KOqzgCgYVWYRiWHSQQWb81o8rNSc0pRWWeAt5sKsUFaeLgoEW8JNQtXn8CeMyVQKWR47grL/Nsaww4REVEHNrV/V3w+cwBCfa68VN5q9sju8NIocbqoCi8uTZNeC9C64JOHbkJCNx+oFTK8OTUen8/sj3AfDWr0Rmw4VoDc0hocyC6FIADv3tcXALD0wHnklFxec7PdsuR8SHcfyCzHzA/p7gMAWGUJTI8MDW/VMv+2xLBDRETkZLQuSjx1axQAoN5ogr+HWtpjSKNSYMnjg5D6yhjcPzAUgiDgrn7mlVPLDp7HmiPmfXoGhHlhXK9ADIv0hcEk4tNtp20+o6beiHWWe2+J8pVeT7CEHQDw1Cil4zAciWGHiIjICU0fHIpQb/OIyrOje0CjaqjtEQQBrqqGKbG7LMvEt58qwo97zdNb1o0OrYXFP+87h1/250AURZRV6/HQl8lIL6iARiXHqGh/6VkDwryhlJtHeZ66NQo6V2Ub9rJ1uBqLiIjICakVcvzv0Ztx+FwZ7rjC8Q3hvm7oF+qJg9mlOFVYCQAYH28+DX5QhDdG9fTDlvQiPP/rYfy8PwcVtQacyK+A1kWBrx4ZCH8PF+lZrio5Xr4jFqcKKjFjcFjbdfAqMOwQERE5qTAfN4T5uLXq3in9uuBgdikAoH+YF4J05p2eBUHApzMG4L87M/HBxlPYd9a8f4+fhxrf/ulmxARdvpnfzIRwu7TfXjiNRURERJgYHwSFzPb4CiuVQobZI7pj43MjcGefYAwM98KvsxOaDDodEUd2iIiICD7uajw+vBu2nyrCXX2Dm7yni6crPpzW/AaDHRXPxgLPxiIiIuqMeDYWERERERh2iIiIyMkx7BAREZFTY9ghIiIip8awQ0RERE6NYYeIiIicGsMOEREROTWGHSIiInJqDDtERETk1Bh2iIiIyKkx7BAREZFTY9ghIiIip8awQ0RERE6NYYeIiIicmsLRDegIRFEEYD4qnoiIiDoH69/b1r/Hm8OwA6CiogIAEBIS4uCWEBER0dWqqKiATqdr9rogXikO3QBMJhNyc3Ph4eEBQRAc3Ry7Ki8vR0hICHJycqDVah3dnDZ3I/X3RuorwP46uxupvzdSX4G27a8oiqioqEBwcDBksuYrcziyA0Amk6Fr166Obkab0mq1N8S/VFY3Un9vpL4C7K+zu5H6eyP1FWi7/rY0omPFAmUiIiJyagw7RERE5NQYdpycWq3G//3f/0GtVju6Ke3iRurvjdRXgP11djdSf2+kvgIdo78sUCYiIiKnxpEdIiIicmoMO0REROTUGHaIiIjIqTHsEBERkVNj2HECCxcuxMCBA+Hh4QF/f3/cddddSE9Pt7mntrYWiYmJ8PHxgbu7O6ZOnYqCggIHtdi+3njjDQiCgGeffVZ6zdn6e/78eTz00EPw8fGBq6sr4uPjsX//fum6KIp45ZVXEBQUBFdXV4wePRqnTp1yYIuvjdFoxMsvv4yIiAi4urqie/fueP31123OvenMfd22bRsmTZqE4OBgCIKA5cuX21xvTd9KSkowffp0aLVaeHp64tFHH0VlZWU79qL1WuqvXq/HCy+8gPj4eLi5uSE4OBgzZ85Ebm6uzTOcpb+Xmj17NgRBwPvvv2/zemfpb2v6evz4cdx5553Q6XRwc3PDwIEDkZ2dLV1vz9/TDDtOICkpCYmJidizZw82bNgAvV6PMWPGoKqqSrpn7ty5WLFiBX755RckJSUhNzcXd999twNbbR/79u3Dp59+it69e9u87kz9vXjxIoYOHQqlUok1a9bg2LFj+Pe//w0vLy/pnrfeegsffvghPvnkEyQnJ8PNzQ1jx45FbW2tA1t+9d58800sXrwY//nPf3D8+HG8+eabeOutt/DRRx9J93TmvlZVVaFPnz5YtGhRk9db07fp06fj6NGj2LBhA1auXIlt27bhiSeeaK8uXJWW+ltdXY0DBw7g5ZdfxoEDB7B06VKkp6fjzjvvtLnPWfrb2LJly7Bnzx4EBwdfdq2z9PdKfT19+jSGDRuG6OhobN26FYcPH8bLL78MFxcX6Z52/T0tktMpLCwUAYhJSUmiKIpiaWmpqFQqxV9++UW65/jx4yIAcffu3Y5q5nWrqKgQo6KixA0bNogjRowQn3nmGVEUna+/L7zwgjhs2LBmr5tMJjEwMFB8++23pddKS0tFtVot/vDDD+3RRLuZOHGi+Kc//cnmtbvvvlucPn26KIrO1VcA4rJly6SfW9O3Y8eOiQDEffv2SfesWbNGFARBPH/+fLu1/Vpc2t+m7N27VwQgZmVliaLonP09d+6c2KVLF/HIkSNiWFiY+N5770nXOmt/m+rr/fffLz700EPNvqe9f09zZMcJlZWVAQC8vb0BACkpKdDr9Rg9erR0T3R0NEJDQ7F7926HtNEeEhMTMXHiRJt+Ac7X3z/++AMDBgzAvffeC39/f/Tr1w+ff/65dD0zMxP5+fk2/dXpdBg0aFCn6++QIUOwadMmnDx5EgBw6NAh7NixA+PHjwfgXH29VGv6tnv3bnh6emLAgAHSPaNHj4ZMJkNycnK7t9neysrKIAgCPD09AThff00mE2bMmIHnn38ecXFxl113lv6aTCasWrUKPXr0wNixY+Hv749BgwbZTHW19+9phh0nYzKZ8Oyzz2Lo0KHo1asXACA/Px8qlUr6BWIVEBCA/Px8B7Ty+v344484cOAAFi5ceNk1Z+vvmTNnsHjxYkRFRWHdunV48skn8fTTT+Obb74BAKlPAQEBNu/rjP3961//igceeADR0dFQKpXo168fnn32WUyfPh2Ac/X1Uq3pW35+Pvz9/W2uKxQKeHt7d/r+19bW4oUXXsC0adOkwyKdrb9vvvkmFAoFnn766SavO0t/CwsLUVlZiTfeeAPjxo3D+vXrMWXKFNx9991ISkoC0P6/p3nquZNJTEzEkSNHsGPHDkc3pc3k5OTgmWeewYYNG2zmf52VyWTCgAEDsGDBAgBAv379cOTIEXzyySd4+OGHHdw6+/r555/x/fffY8mSJYiLi0NqaiqeffZZBAcHO11fqYFer8d9990HURSxePFiRzenTaSkpOCDDz7AgQMHIAiCo5vTpkwmEwBg8uTJmDt3LgCgb9++2LVrFz755BOMGDGi3dvEkR0nMmfOHKxcuRJbtmxB165dpdcDAwNRX1+P0tJSm/sLCgoQGBjYzq28fikpKSgsLMRNN90EhUIBhUKBpKQkfPjhh1AoFAgICHCq/gYFBSE2NtbmtZiYGGlVg7VPl65i6Iz9ff7556XRnfj4eMyYMQNz586VRvCcqa+Xak3fAgMDUVhYaHPdYDCgpKSk0/bfGnSysrKwYcMGaVQHcK7+bt++HYWFhQgNDZV+b2VlZeG5555DeHg4AOfpr6+vLxQKxRV/b7Xn72mGHScgiiLmzJmDZcuWYfPmzYiIiLC53r9/fyiVSmzatEl6LT09HdnZ2UhISGjv5l632267DWlpaUhNTZW+BgwYgOnTp0vfO1N/hw4detlWAidPnkRYWBgAICIiAoGBgTb9LS8vR3Jycqfrb3V1NWQy219Lcrlc+i9FZ+rrpVrTt4SEBJSWliIlJUW6Z/PmzTCZTBg0aFC7t/l6WYPOqVOnsHHjRvj4+Nhcd6b+zpgxA4cPH7b5vRUcHIznn38e69atA+A8/VWpVBg4cGCLv7fa/e8lu5c8U7t78sknRZ1OJ27dulXMy8uTvqqrq6V7Zs+eLYaGhoqbN28W9+/fLyYkJIgJCQkObLV9NV6NJYrO1d+9e/eKCoVC/Ne//iWeOnVK/P7770WNRiN+99130j1vvPGG6OnpKf7+++/i4cOHxcmTJ4sRERFiTU2NA1t+9R5++GGxS5cu4sqVK8XMzExx6dKloq+vr/iXv/xFuqcz97WiokI8ePCgePDgQRGA+O6774oHDx6UVh+1pm/jxo0T+/XrJyYnJ4s7duwQo6KixGnTpjmqSy1qqb/19fXinXfeKXbt2lVMTU21+d1VV1cnPcNZ+tuUS1djiWLn6e+V+rp06VJRqVSKn332mXjq1Cnxo48+EuVyubh9+3bpGe35e5phxwkAaPLrq6++ku6pqakR//znP4teXl6iRqMRp0yZIubl5Tmu0XZ2adhxtv6uWLFC7NWrl6hWq8Xo6Gjxs88+s7luMpnEl19+WQwICBDVarV42223ienp6Q5q7bUrLy8Xn3nmGTE0NFR0cXERu3XrJv7973+3+cuvM/d1y5YtTf67+vDDD4ui2Lq+FRcXi9OmTRPd3d1FrVYrPvLII2JFRYUDenNlLfU3MzOz2d9dW7ZskZ7hLP1tSlNhp7P0tzV9/fLLL8XIyEjRxcVF7NOnj7h8+XKbZ7Tn72lBFBttTUpERETkZFizQ0RERE6NYYeIiIicGsMOEREROTWGHSIiInJqDDtERETk1Bh2iIiIyKkx7BAREZFTY9ghIiIip8awQ0QOER4ejvfff7/V92/duhWCIFx2cCAR0ZVwB2UiapWRI0eib9++VxVQWlJUVAQ3NzdoNJpW3V9fX4+SkhIEBARAEAS7tOFqbd26FaNGjcLFixfh6enpkDYQ0dVTOLoBROQ8RFGE0WiEQnHlXy1+fn5X9WyVSoXAwMBrbRoR3cA4jUVEVzRr1iwkJSXhgw8+gCAIEAQBZ8+elaaW1qxZg/79+0OtVmPHjh04ffo0Jk+ejICAALi7u2PgwIHYuHGjzTMvncYSBAFffPEFpkyZAo1Gg6ioKPzxxx/S9Uunsb7++mt4enpi3bp1iImJgbu7O8aNG4e8vDzpPQaDAU8//TQ8PT3h4+ODF154AQ8//DDuuuuuZvualZWFSZMmwcvLC25uboiLi8Pq1atx9uxZjBo1CgDg5eUFQRAwa9YsAIDJZMLChQsREREBV1dX9OnTB7/++utlbV+1ahV69+4NFxcXDB48GEeOHLni5xLR9WPYIaIr+uCDD5CQkIDHH38ceXl5yMvLQ0hIiHT9r3/9K9544w0cP34cvXv3RmVlJSZMmIBNmzbh4MGDGDduHCZNmoTs7OwWP+cf//gH7rvvPhw+fBgTJkzA9OnTUVJS0uz91dXVeOedd/C///0P27ZtQ3Z2NubPny9df/PNN/H999/jq6++ws6dO1FeXo7ly5e32IbExETU1dVh27ZtSEtLw5tvvgl3d3eEhITgt99+AwCkp6cjLy8PH3zwAQBg4cKF+Pbbb/HJJ5/g6NGjmDt3Lh566CEkJSXZPPv555/Hv//9b+zbtw9+fn6YNGkS9Hp9i59LRHbQJmepE5HTGTFihPjMM8/YvLZlyxYRgLh8+fIrvj8uLk786KOPpJ/DwsLE9957T/oZgPjSSy9JP1dWVooAxDVr1th81sWLF0VRFMWvvvpKBCBmZGRI71m0aJEYEBAg/RwQECC+/fbb0s8Gg0EMDQ0VJ0+e3Gw74+PjxVdffbXJa5e2QRRFsba2VtRoNOKuXbts7n300UfFadOm2bzvxx9/lK4XFxeLrq6u4k8//XTFzyWi68OaHSK6bgMGDLD5ubKyEq+++ipWrVqFvLw8GAwG1NTUXHFkp3fv3tL3bm5u0Gq1KCwsbPZ+jUaD7t27Sz8HBQVJ95eVlaGgoAA333yzdF0ul6N///4wmUzNPvPpp5/Gk08+ifXr12P06NGYOnWqTbsulZGRgerqatx+++02r9fX16Nfv342ryUkJEjfe3t7o2fPnjh+/Pg1fS4RtR6nsYjourm5udn8PH/+fCxbtgwLFizA9u3bkZqaivj4eNTX17f4HKVSafOzIAgtBpOm7hevc4HpY489hjNnzmDGjBlIS0vDgAED8NFHHzV7f2VlJQBg1apVSE1Nlb6OHTtmU7dj788lotZj2CGiVlGpVDAaja26d+fOnZg1axamTJmC+Ph4BAYG4uzZs23bwEvodDoEBARg37590mtGoxEHDhy44ntDQkIwe/ZsLF26FM899xw+//xzAOY/A+tzrGJjY6FWq5GdnY3IyEibr8Z1TQCwZ88e6fuLFy/i5MmTiImJueLnEtH14TQWEbVKeHg4kpOTcfbsWbi7u8Pb27vZe6OiorB06VJMmjQJgiDg5ZdfbnGEpq089dRTWLhwISIjIxEdHY2PPvoIFy9ebHGfnmeffRbjx49Hjx49cPHiRWzZskUKJGFhYRAEAStXrsSECRPg6uoKDw8PzJ8/H3PnzoXJZMKwYcNQVlaGnTt3QqvV4uGHH5ae/dprr8HHxwcBAQH4+9//Dl9fX2llWEufS0TXhyM7RNQq8+fPh1wuR2xsLPz8/Fqsv3n33Xfh5eWFIUOGYNKkSRg7dixuuummdmyt2QsvvIBp06Zh5syZSEhIgLu7O8aOHQsXF5dm32M0GpGYmIiYmBiMGzcOPXr0wMcffwwA6NKlC/7xj3/gr3/9KwICAjBnzhwAwOuvv46XX34ZCxculN63atUqRERE2Dz7jTfewDPPPIP+/fsjPz8fK1assBktau5ziej6cAdlIrphmEwmxMTE4L777sPrr7/ebp/LnZeJHIvTWETktLKysrB+/XqMGDECdXV1+M9//oPMzEw8+OCDjm4aEbUjTmMRkdOSyWT4+uuvMXDgQAwdOhRpaWnYuHEja2GIbjCcxiIiIiKnxpEdIiIicmoMO0REROTUGHaIiIjIqTHsEBERkVNj2CEiIiKnxrBDRERETo1hh4iIiJwaww4RERE5tf8PIuu2NxL5a+kAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 准备优化器和模型\n",
    "vocab_size = len(dataset.token2id)\n",
    "hidden_size = 32\n",
    "n_layers = 1\n",
    "dropout = 0\n",
    "n_tags = len(dataset.label2id)\n",
    "batch_size = 128\n",
    "epochs = 20\n",
    "learning_rate = 1e-2\n",
    "\n",
    "lstm_crf = LSTM_CRF(vocab_size, hidden_size, n_layers, dropout, n_tags)\n",
    "train(lstm_crf, batch_size, epochs, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8a9138a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.5659434992976432, recall = 0.43228421554601815, f1 = 0.4901655964852991\n",
      "precision = 0.4885057471264368, recall = 0.3643600734843846, f1 = 0.41739740441950196\n"
     ]
    }
   ],
   "source": [
    "evaluate(train_X, train_Y, lstm_crf, batch_size)\n",
    "evaluate(test_X, test_Y, lstm_crf, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d42334d8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c135f708",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
